diff --git a/BUILD.bazel b/BUILD.bazel index 7d17bb3f..982b47cd 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -220,6 +220,30 @@ cc_test( ], ) +genrule( + name = "packed_tokenizer", + srcs = ["tokenizer/testdata/tokenizer.json"], + outs = ["tokenizer/testdata/tokenizer.packed"], + cmd = "$(location //python:convert_from_safetensors) --pack_tokenizer --tokenizer_file $(location tokenizer/testdata/tokenizer.json) --output_packed $@", + tools = ["//python:convert_from_safetensors"], +) + +cc_test( + name = "bpe_tokenizer_test", + srcs = ["tokenizer/bpe_tokenizer_test.cc"], + data = [ + "testdata/frankenstein.txt", + "tokenizer/testdata/tokenizer.json", + "tokenizer/testdata/tokenizer.model", + ":packed_tokenizer", + ], + deps = [ + ":tokenizer", + "//testing/base/public:gunit_main", # buildcleaner: keep + "//io", + ], +) + cc_library( name = "tensor_info", srcs = [ @@ -252,10 +276,21 @@ cc_library( cc_library( name = "tokenizer", - srcs = ["gemma/tokenizer.cc"], - hdrs = ["gemma/tokenizer.h"], + srcs = [ + "gemma/tokenizer.cc", + "tokenizer/bpe_tokenizer.cc", + "tokenizer/sentencepiece_tokenizer.cc", + ], + hdrs = [ + "gemma/tokenizer.h", + "tokenizer/bpe_tokenizer.h", + "tokenizer/sentencepiece_tokenizer.h", + "tokenizer/tokenizer.h", + ], deps = [ + ":basics", ":configs", + "//io:fields", "@highway//:hwy", "@highway//:profiler", "@com_google_sentencepiece//:sentencepiece_processor", diff --git a/CMakeLists.txt b/CMakeLists.txt index f7a678ea..552f1aaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -192,6 +192,11 @@ set(SOURCES paligemma/image.h paligemma/paligemma_helper.cc paligemma/paligemma_helper.h + tokenizer/bpe_tokenizer.cc + tokenizer/bpe_tokenizer.h + tokenizer/sentencepiece_tokenizer.cc + tokenizer/sentencepiece_tokenizer.h + tokenizer/tokenizer.h util/allocator.cc util/allocator.h util/basics.cc diff --git a/compression/python/compression_clif_aux.cc b/compression/python/compression_clif_aux.cc index 3568ad34..6cf91c7a 100644 --- a/compression/python/compression_clif_aux.cc +++ b/compression/python/compression_clif_aux.cc @@ -45,6 +45,7 @@ #include "hwy/highway.h" // After highway.h #include "compression/compress-inl.h" +#include "tokenizer/bpe_tokenizer.h" // SIMD code, compiled once per target. HWY_BEFORE_NAMESPACE(); @@ -118,11 +119,18 @@ class SbsWriterImpl : public ISbsWriter { } void Write(const ModelConfig& config, - const std::string& tokenizer_path) override { - const GemmaTokenizer tokenizer( - tokenizer_path.empty() ? kMockTokenizer - : ReadFileToString(Path(tokenizer_path))); - WriteSingleFile(config, tokenizer, serialized_mat_ptrs_, writer_); + const std::string& tokenizer_blob) override { + WriteSingleFile(config, BuildTokenizer(config, tokenizer_blob), + serialized_mat_ptrs_, writer_); + } + + static GemmaTokenizer BuildTokenizer(const ModelConfig& config, + const std::string& tokenizer_blob) { + if (tokenizer_blob.empty()) return GemmaTokenizer(kMockTokenizer); + if (config.tokenizer_kind == TokenizerKind::kHfBpe) { + return GemmaTokenizer(CreateBpeTokenizer(tokenizer_blob)); + } + return GemmaTokenizer(tokenizer_blob); } ThreadingContext ctx_; diff --git a/compression/python/compression_clif_aux.h b/compression/python/compression_clif_aux.h index 69798652..2b2953fb 100644 --- a/compression/python/compression_clif_aux.h +++ b/compression/python/compression_clif_aux.h @@ -44,7 +44,7 @@ class ISbsWriter { const TensorInfo& tensor_info) = 0; virtual void Write(const ModelConfig& config, - const std::string& tokenizer_path) = 0; + const std::string& tokenizer_blob) = 0; }; // Non-virtual class used by pybind that calls the interface's virtual methods. @@ -58,8 +58,8 @@ class SbsWriter { impl_->Insert(name, weights, type, tensor_info); } - void Write(const ModelConfig& config, const std::string& tokenizer_path) { - impl_->Write(config, tokenizer_path); + void Write(const ModelConfig& config, const std::string& tokenizer_blob) { + impl_->Write(config, tokenizer_blob); } private: diff --git a/compression/python/compression_extension.cc b/compression/python/compression_extension.cc index e3d15569..c5da385c 100644 --- a/compression/python/compression_extension.cc +++ b/compression/python/compression_extension.cc @@ -46,7 +46,7 @@ PYBIND11_MODULE(compression, m) { class_(m, "SbsWriter") .def(init()) .def("insert", CallWithF32Span<&SbsWriter::Insert>) - .def("write", &SbsWriter::Write, arg("config"), arg("tokenizer_path")); + .def("write", &SbsWriter::Write, arg("config"), arg("tokenizer_blob")); class_(m, "MatPtr") // No init, only created within C++. diff --git a/compression/python/compression_test.py b/compression/python/compression_test.py index 16e6bf9a..34a7d5ac 100644 --- a/compression/python/compression_test.py +++ b/compression/python/compression_test.py @@ -102,8 +102,8 @@ def test_sbs_writer(self): configs.Type.kSFP, configs.PromptWrapping.GEMMA_IT, ) - tokenizer_path = "" # no tokenizer required for testing - writer.write(config, tokenizer_path) + tokenizer_blob = "" # no tokenizer required for testing + writer.write(config, tokenizer_blob) print("Ignore next two warnings; test does not enable model deduction.") reader = compression.SbsReader(temp_file.full_path) diff --git a/gemma/configs.h b/gemma/configs.h index 201fc4d3..4386c2cb 100644 --- a/gemma/configs.h +++ b/gemma/configs.h @@ -52,6 +52,18 @@ enum class PromptWrapping { kSentinel // must be last }; +// Which tokenizer engine the `.sbs` tokenizer blob targets. +enum class TokenizerKind { + kSentencePiece = 0, // blob is a SentencePiece proto + kHfBpe = 1, // blob is a compact BPE format built from HF `tokenizer.json` + kSentinel // must be last +}; + +static inline bool EnumValid(TokenizerKind kind) { + return static_cast(kind) < + static_cast(TokenizerKind::kSentinel); +} + // This is used in `ModelConfig.Specifier`, so the strings will not change, // though new ones may be added. static inline const char* WrappingSuffix(PromptWrapping wrapping) { @@ -605,6 +617,8 @@ struct ModelConfig : public IFields { visitor(hc_eps); visitor(num_mtp_layers); + visitor(tokenizer_kind); + // Append new fields here, then update `python/configs.cc`. } @@ -760,6 +774,8 @@ struct ModelConfig : public IFields { // is an extra transformer layer used for speculative decoding; it is not // counted in `num_layers` and never runs as part of the main stack. uint32_t num_mtp_layers = 0; + + TokenizerKind tokenizer_kind = TokenizerKind::kSentencePiece; }; // Returns the sub-config for the ViT model of the PaliGemma model. diff --git a/gemma/model_store.cc b/gemma/model_store.cc index 67aa3068..a83c38a6 100644 --- a/gemma/model_store.cc +++ b/gemma/model_store.cc @@ -34,6 +34,7 @@ #include "io/blob_store.h" #include "io/fields.h" #include "io/io.h" // Path +#include "tokenizer/bpe_tokenizer.h" #include "util/basics.h" #include "util/threading_context.h" #include "hwy/base.h" @@ -58,12 +59,8 @@ static void WarnIfExtra(const IFields::ReadResult& result, const char* name) { } } -// Returns the serialized tokenizer (std::string is required for proto). -// Reads it from a blob or from a separate file if pre-2025. -static std::string ReadTokenizer(BlobReader& reader, - const Path& tokenizer_path) { - PROFILER_ZONE("Startup.ReadTokenizer"); - +// Reads the raw `tokenizer` blob bytes, or empty if absent. +static std::string ReadTokenizerBlob(BlobReader& reader) { std::string tokenizer; // Check prevents `CallWithSpan` from printing a warning. if (reader.Find(kTokenizerName)) { @@ -76,27 +73,51 @@ static std::string ReadTokenizer(BlobReader& reader, "instead specify a tokenizer file via --tokenizer."); } } + return tokenizer; +} + +// Constructs the tokenizer. +static GemmaTokenizer ReadTokenizer(BlobReader& reader, + const Path& tokenizer_path, + const ModelConfig& config) { + PROFILER_ZONE("Startup.ReadTokenizer"); + + if (config.tokenizer_kind == TokenizerKind::kHfBpe) { + const std::string blob = ReadTokenizerBlob(reader); + if (!blob.empty() && blob != kMockTokenizer) { + return GemmaTokenizer(CreateBpeTokenizer(blob)); + } + if (!tokenizer_path.Empty()) { + HWY_WARN("No separate tokenizer file is supported for HF BPE models. " + "Tests may continue but inference will fail. %s.", + tokenizer_path.path.c_str()); + return GemmaTokenizer(kMockTokenizer); + } + HWY_WARN("BlobStore has no HuggingFace BPE tokenizer specified. Tests may " + "continue but inference will fail."); + return GemmaTokenizer(kMockTokenizer); + } - // Read actual tokenizer from blob. - if (!tokenizer.empty() && tokenizer != kMockTokenizer) { + // SentencePiece: the blob is a SentencePiece proto. + const std::string blob = ReadTokenizerBlob(reader); + if (!blob.empty() && blob != kMockTokenizer) { if (!tokenizer_path.Empty()) { HWY_WARN("--weights has tokenizer but overriding with %s.", tokenizer_path.path.c_str()); - return ReadFileToString(tokenizer_path); + return GemmaTokenizer(ReadFileToString(tokenizer_path)); } - - return tokenizer; + return GemmaTokenizer(blob); } // No blob but user specified path to file: read it or abort. if (!tokenizer_path.Empty()) { - return ReadFileToString(tokenizer_path); + return GemmaTokenizer(ReadFileToString(tokenizer_path)); } HWY_WARN( "BlobStore does not contain a tokenizer and no --tokenizer was " "specified. Tests may continue but inference will fail."); - return kMockTokenizer; + return GemmaTokenizer(kMockTokenizer); } using KeyVec = std::vector; @@ -393,7 +414,7 @@ void ModelStore::CreateMatPtrs(BlobReader& reader) { ModelStore::ModelStore(BlobReader& reader, const Path& tokenizer_path, Tristate wrapping) : config_(ReadOrDeduceConfig(reader, wrapping)), - tokenizer_(ReadTokenizer(reader, tokenizer_path)) { + tokenizer_(ReadTokenizer(reader, tokenizer_path, config_)) { if (!ReadMatPtrs(reader)) { // Pre-2025 format. CreateMatPtrs(reader); scales_ = ReadScales(reader, config_); diff --git a/gemma/tokenizer.cc b/gemma/tokenizer.cc index b48fbf0e..a6b7b273 100644 --- a/gemma/tokenizer.cc +++ b/gemma/tokenizer.cc @@ -19,89 +19,49 @@ #include #include +#include #include #include "gemma/configs.h" // PromptWrapping -#include "hwy/base.h" // HWY_ASSERT -#include "hwy/profiler.h" -// copybara:import_next_line:sentencepiece -#include "src/sentencepiece_processor.h" +#include "tokenizer/sentencepiece_tokenizer.h" namespace gcpp { // Set this to true to debug tokenizer tokens. constexpr bool kShowTokenization = false; -class GemmaTokenizer::Impl { - public: - Impl() = default; - // Loads the tokenizer from a serialized proto. - explicit Impl(const std::string& tokenizer_proto) { - if (tokenizer_proto == kMockTokenizer) return; - PROFILER_ZONE("Startup.tokenizer"); - spp_ = std::make_unique(); - if (!spp_->LoadFromSerializedProto(tokenizer_proto).ok()) { - HWY_ABORT("Failed to load tokenizer from %zu byte serialized proto.", - tokenizer_proto.size()); - } - } - - std::string Serialize() const { - return spp_ ? spp_->serialized_model_proto() : kMockTokenizer; - } - - bool Encode(const std::string& input, - std::vector* pieces) const { - return spp_ && spp_->Encode(input, pieces).ok(); - } - - bool Encode(const std::string& input, std::vector* ids) const { - if constexpr (kShowTokenization) { - bool is_ok = spp_ && spp_->Encode(input, ids).ok(); - for (int i = 0; i < static_cast(ids->size()); i++) { - fprintf(stderr, "%3d: %d\n", i, (*ids)[i]); - } - return is_ok; - } else { - return spp_ && spp_->Encode(input, ids).ok(); - } - } - - // Given a sequence of ids, decodes it into a detokenized output. - bool Decode(const std::vector& ids, std::string* detokenized) const { - return spp_ && spp_->Decode(ids, detokenized).ok(); - } - - private: - std::unique_ptr spp_; -}; - -GemmaTokenizer::GemmaTokenizer(const std::string& tokenizer_proto) - : impl_(std::make_unique(tokenizer_proto)) { - HWY_ASSERT(impl_); +GemmaTokenizer::GemmaTokenizer(const std::string& tokenizer_proto) { + if (tokenizer_proto == kMockTokenizer) return; + impl_ = CreateSentencePieceTokenizer(tokenizer_proto); } +GemmaTokenizer::GemmaTokenizer(std::unique_ptr impl) + : impl_(std::move(impl)) {} + // Default suffices, but they must be defined after GemmaTokenizer::Impl. GemmaTokenizer::~GemmaTokenizer() = default; GemmaTokenizer::GemmaTokenizer(GemmaTokenizer&& other) = default; GemmaTokenizer& GemmaTokenizer::operator=(GemmaTokenizer&& other) = default; -std::string GemmaTokenizer::Serialize() const { return impl_->Serialize(); } - -bool GemmaTokenizer::Encode(const std::string& input, - std::vector* pieces) const { - return impl_->Encode(input, pieces); +std::string GemmaTokenizer::Serialize() const { + return impl_ ? impl_->Serialize() : kMockTokenizer; } bool GemmaTokenizer::Encode(const std::string& input, std::vector* ids) const { - return impl_->Encode(input, ids); + if (!impl_ || !impl_->Encode(input, *ids)) return false; + if constexpr (kShowTokenization) { + for (int i = 0; i < static_cast(ids->size()); i++) { + fprintf(stderr, "%3d: %d\n", i, (*ids)[i]); + } + } + return true; } // Given a sequence of ids, decodes it into a detokenized output. bool GemmaTokenizer::Decode(const std::vector& ids, std::string* detokenized) const { - return impl_->Decode(ids, detokenized); + return impl_ && impl_->Decode(ids, *detokenized); } // Negligible CPU time in the ctor body. diff --git a/gemma/tokenizer.h b/gemma/tokenizer.h index aca01f9c..89eaf4b2 100644 --- a/gemma/tokenizer.h +++ b/gemma/tokenizer.h @@ -26,6 +26,8 @@ namespace gcpp { +class Tokenizer; + constexpr int BOS_ID = 2; // beginning of sequence // To avoid the complexity of storing the tokenizer into testdata/ or @@ -36,23 +38,23 @@ constexpr const char* kMockTokenizer = "unavailable"; class GemmaTokenizer { // These must be defined after the definition of `Impl`. public: - // If unavailable, pass `kMockTokenizer`. + // SentencePiece backend. If unavailable, pass `kMockTokenizer`. explicit GemmaTokenizer(const std::string& tokenizer_proto); ~GemmaTokenizer(); GemmaTokenizer(GemmaTokenizer&& other); GemmaTokenizer& operator=(GemmaTokenizer&& other); + explicit GemmaTokenizer(std::unique_ptr impl); + // Returns `kMockTokenizer` if unavailable. std::string Serialize() const; // Returns false on failure or if unavailable. - bool Encode(const std::string& input, std::vector* pieces) const; bool Encode(const std::string& input, std::vector* ids) const; bool Decode(const std::vector& ids, std::string* detokenized) const; private: - class Impl; - std::unique_ptr impl_; + std::unique_ptr impl_; }; class GemmaChatTemplate { diff --git a/io/fields.cc b/io/fields.cc index 4516e891..6b54a465 100644 --- a/io/fields.cc +++ b/io/fields.cc @@ -57,7 +57,7 @@ class VisitorBase : public IFieldsVisitor { // Return bool to avoid having to check AnyInvalid() after calling. bool CheckStringLength(uint32_t num_u32) { // Disallow long strings for safety, and to prevent them being used for - // arbitrary data (we also require them to be ASCII). + // arbitrary data. if (HWY_UNLIKELY(num_u32 > 64)) { NotifyInvalid("String num_u32=%u too large\n", num_u32); return false; @@ -67,8 +67,8 @@ class VisitorBase : public IFieldsVisitor { bool CheckStringU32(uint32_t u32, uint32_t i, uint32_t num_u32) { // Although strings are zero-padded to u32, an entire u32 should not be - // zero, and upper bits should not be set (ASCII-only). - if (HWY_UNLIKELY(u32 == 0 || (u32 & 0x80808080))) { + // zero. + if (HWY_UNLIKELY(u32 == 0)) { NotifyInvalid("Invalid characters %x at %u of %u\n", u32, i, num_u32); return false; } diff --git a/io/fields.h b/io/fields.h index 25728aac..0aaf6eff 100644 --- a/io/fields.h +++ b/io/fields.h @@ -105,7 +105,7 @@ class IFieldsVisitor { uint32_t num = static_cast(value.size()); operator()(num); - if (HWY_UNLIKELY(num > 64 * 1024)) { + if (HWY_UNLIKELY(num > 8 * 1024 * 1024)) { return NotifyInvalid("Vector too long %u\n", num); } diff --git a/io/fields_test.cc b/io/fields_test.cc index f720c158..34a24f12 100644 --- a/io/fields_test.cc +++ b/io/fields_test.cc @@ -287,16 +287,19 @@ TEST(FieldsTest, TestInvalidString) { // Too long new_fields.new_str.assign(257, 'a'); EXPECT_TRUE(new_fields.Write().empty()); +} - // First byte not ASCII - new_fields.new_str.assign("123"); - new_fields.new_str[0] = 128; - EXPECT_TRUE(new_fields.Write().empty()); +// Verify non-ASCII strings are accepted. +TEST(FieldsTest, TestNonAsciiString) { + NewFields new_fields; + new_fields.new_str = "你好世界"; // "Hello World" in Chinese + const std::vector storage = new_fields.Write(); + EXPECT_FALSE(storage.empty()); - // Upper byte in later u32 not ASCII - new_fields.new_str.assign("ABCDEFGH"); - new_fields.new_str[7] = 255; - EXPECT_TRUE(new_fields.Write().empty()); + NewFields copy; + const ReadResult result = copy.Read(Span(storage), 0); + CheckConsumedAll(result, storage.size()); + EXPECT_EQ("你好世界", copy.new_str); } // Write two structs to the same storage. diff --git a/python/configs.cc b/python/configs.cc index b2708b44..aa3ad27d 100644 --- a/python/configs.cc +++ b/python/configs.cc @@ -81,6 +81,10 @@ PYBIND11_MODULE(configs, py_module) { enum_(py_module, "ResidualType") .value("Add", gcpp::ResidualType::Add); + enum_(py_module, "TokenizerKind") + .value("kSentencePiece", gcpp::TokenizerKind::kSentencePiece) + .value("kHfBpe", gcpp::TokenizerKind::kHfBpe); + enum_(py_module, "Model") .value("UNKNOWN", gcpp::Model::UNKNOWN) .value("CUSTOM", gcpp::Model::CUSTOM) @@ -226,6 +230,7 @@ PYBIND11_MODULE(configs, py_module) { .def_readwrite("hc_sinkhorn_iters", &gcpp::ModelConfig::hc_sinkhorn_iters) .def_readwrite("hc_eps", &gcpp::ModelConfig::hc_eps) .def_readwrite("num_mtp_layers", &gcpp::ModelConfig::num_mtp_layers) + .def_readwrite("tokenizer_kind", &gcpp::ModelConfig::tokenizer_kind) .def("add_layer_config", &gcpp::ModelConfig::AddLayerConfig, arg("layer_config")) diff --git a/python/convert_from_safetensors.py b/python/convert_from_safetensors.py index 89e5e629..518c030d 100644 --- a/python/convert_from_safetensors.py +++ b/python/convert_from_safetensors.py @@ -70,6 +70,90 @@ def compute_scale(x: np.ndarray) -> float: return max(1.0, magnitude / 1.875) +def pack_bpe_tokenizer(tokenizer_json_path: str) -> bytes: + """Packs a HuggingFace tokenizer.json into compact binary format.""" + import struct # pylint: disable=g-import-not-at-top + + with open(tokenizer_json_path, "r", encoding="utf-8") as f: + j = json.load(f) + + model = j.get("model", {}) + if model.get("type") != "BPE": + raise ValueError("tokenizer.json model.type must be 'BPE'") + + vocab = model.get("vocab", {}) + merges = model.get("merges", []) + added_tokens = j.get("added_tokens", []) + + vocab_map = {} + max_id = max(vocab.values()) if vocab else 0 + id_to_token = [""] * (max_id + 1) + for token, tid in vocab.items(): + vocab_map[token] = tid + id_to_token[tid] = token + + added_ids = [] + for at in added_tokens: + content = at.get("content") + tid = at.get("id") + if not content: + continue + vocab_map[content] = tid + if tid >= len(id_to_token): + id_to_token.extend([""] * (tid - len(id_to_token) + 1)) + id_to_token[tid] = content + added_ids.append(tid) + added_ids.sort() + + merge_ranks = [] + for rank, m in enumerate(merges): + if not isinstance(m, list) or len(m) != 2: + raise ValueError(f"merges[{rank}] is not a [left, right] array") + + left, right = m + if left in vocab_map and right in vocab_map and (left + right) in vocab_map: + merge_ranks.append((rank, vocab_map[left], vocab_map[right])) + + bpe_flags = 1 << 1 # kFlagSpaceReplace + if any(f"<0x{b:02X}>" in vocab_map for b in range(256)): + bpe_flags |= 1 << 0 # kFlagByteFallback + + unk = model.get("unk_token", "") + unk_id = vocab_map.get(unk, 0) + + by_rank = sorted(merge_ranks, key=lambda x: x[0]) + + payload_words = [] + payload_words.append(bpe_flags) + payload_words.append(unk_id) + + # vocab (vector of strings) + payload_words.append(len(id_to_token)) + for token in id_to_token: + token_bytes = token.encode("utf-8") + num_u32 = (len(token_bytes) + 3) // 4 + payload_words.append(num_u32) + padded_bytes = token_bytes + b"\0" * (num_u32 * 4 - len(token_bytes)) + for i in range(num_u32): + u32 = struct.unpack("=I", padded_bytes[i*4:(i+1)*4])[0] + payload_words.append(u32) + + # merges (vector of u32) + payload_words.append(len(by_rank) * 2) + for _, left_id, right_id in by_rank: + payload_words.append(left_id) + payload_words.append(right_id) + + # added_ids (vector of u32) + payload_words.append(len(added_ids)) + payload_words.extend(added_ids) + + # IFields header is just the count of following u32s! + total_words = [len(payload_words)] + payload_words + + return struct.pack(f"={len(total_words)}I", *total_words) + + def _is_float_param(param_name: str) -> bool: """Returns whether the tensor should be stored as float32.""" for prefix in [ @@ -88,7 +172,14 @@ def _is_float_param(param_name: str) -> bool: def _is_bf16_param(param_name: str) -> bool: """Returns whether the tensor should be stored as bf16.""" - for prefix in ["pre_", "post_", "c_", "img_head_kernel"]: + for prefix in [ + "pre_", + "post_", + "c_", + "img_head_kernel", + "query_norm", + "key_norm", + ]: if param_name.startswith(prefix): return True return False @@ -512,7 +603,9 @@ def add_vit_qkv_einsum(i): # Write everything to the sbs file. assert model_specifier.startswith("paligemma") sbs_config = configs.ModelConfig(model_specifier) - writer.write(sbs_config, tokenizer_file) + with open(tokenizer_file, "rb") as f: + tokenizer_blob = f.read() + writer.write(sbs_config, tokenizer_blob) # Write the metadata for manual inspection. with open(csv_file, "w") as csv_handle: @@ -736,7 +829,14 @@ def add_gating_einsum(i): print(f"WARNING: leftover params not consumed: {list(params.keys())[:10]}") sbs_config = configs.ModelConfig(model_specifier) - writer.write(sbs_config, tokenizer_file) + if tokenizer_file.endswith(".json"): + sbs_config.tokenizer_kind = configs.TokenizerKind.kHfBpe + tokenizer_blob = pack_bpe_tokenizer(tokenizer_file) + else: + sbs_config.tokenizer_kind = configs.TokenizerKind.kSentencePiece + with open(tokenizer_file, "rb") as f: + tokenizer_blob = f.read() + writer.write(sbs_config, tokenizer_blob) with open(csv_file, "w") as csv_handle: csv.writer(csv_handle).writerows(metadata) @@ -745,15 +845,14 @@ def add_gating_einsum(i): _MODEL_SPECIFIER = flags.DEFINE_string( "model_specifier", None, - "String specifying model, size, weight, wrapping (ModelConfig.Specifier)", - required=True, + "String specifying model, size, weight, wrapping (ModelConfig.Specifier)" + " (Required)", ) _LOAD_PATH = flags.DEFINE_string( "load_path", None, - "Path to the safetensors index.json file to read", - required=True, + "Path to the safetensors index.json file to read (Required)", ) _TOKENIZER_FILE = flags.DEFINE_string( "tokenizer_file", @@ -771,12 +870,41 @@ def add_gating_einsum(i): "Path to the sbs file to write", ) +_PACK_TOKENIZER = flags.DEFINE_bool( + "pack_tokenizer", + False, + "If true, only pack the tokenizer provided in --tokenizer_file and exit.", + required=False, +) +_OUTPUT_PACKED = flags.DEFINE_string( + "output_packed", + None, + "Path to write the packed tokenizer file (used with --pack_tokenizer)", + required=False, +) + def main(argv: Sequence[str]) -> None: if len(argv) > 1: raise app.UsageError("Too many command-line arguments.") logging.use_python_logging() logging.set_verbosity(logging.INFO) + + if _PACK_TOKENIZER.value: + if not _TOKENIZER_FILE.value: + raise app.UsageError("--tokenizer_file is required for --pack_tokenizer") + tokenizer_blob = pack_bpe_tokenizer(_TOKENIZER_FILE.value) + out_file = _OUTPUT_PACKED.value or (_TOKENIZER_FILE.value + ".packed") + with open(out_file, "wb") as f: + f.write(tokenizer_blob) + logging.info("Packed tokenizer written to %s", out_file) + return + + if not _MODEL_SPECIFIER.value or not _LOAD_PATH.value: + raise app.UsageError( + "Missing required flags: --model_specifier and --load_path" + ) + model_specifier = _MODEL_SPECIFIER.value load_path = _LOAD_PATH.value tokenizer_file = _TOKENIZER_FILE.value diff --git a/tokenizer/bpe_tokenizer.cc b/tokenizer/bpe_tokenizer.cc new file mode 100644 index 00000000..d901e3ca --- /dev/null +++ b/tokenizer/bpe_tokenizer.cc @@ -0,0 +1,489 @@ +// Copyright 2024 Google LLC +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tokenizer/bpe_tokenizer.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io/fields.h" // IFields +#include "tokenizer/tokenizer.h" +#include "util/basics.h" // HWY_ABORT +#include "hwy/profiler.h" +namespace gcpp { + +namespace { + +// The whitespace replacement character "▁" (U+2581, "lower one eighth block"). +constexpr const char* kSpaceRepl = "\xe2\x96\x81"; + +// Bit flags stored in `BpeTokenizerBlob::flags`. Currently informational +// (the corresponding behavior is fixed); reserved so readers can branch later. +constexpr uint32_t kFlagByteFallback = 1u << 0; +constexpr uint32_t kFlagSpaceReplace = 1u << 1; + + +class BpeTokenizer : public Tokenizer { + public: + BpeTokenizer() = default; + + std::string Serialize() const override; + + bool Deserialize(std::string_view data) override; + + bool Encode(std::string_view input, std::vector& ids) const override; + + bool Decode(hwy::Span ids, + std::string& detokenized) const override; + + private: + // Result of a single vocabulary merge: where to merge and what it becomes. + struct MergeRule { + int rank; + int new_id; + }; + + void LoadByteTokens(); + const std::string& IdToToken(int id) const; + int IdByteValue(int id) const; + int MatchAddedToken(std::string_view input, size_t i, size_t* len) const; + void SplitOnAddedTokens(std::string_view input, + std::vector* ids) const; + std::string Normalize(std::string_view text) const; + void EncodeSpan(std::string_view text, std::vector* ids) const; + void MergeSymbols(std::vector* sym_id) const; + + std::unordered_map vocab_; + std::vector id_to_token_; + std::unordered_map merge_ranks_; + std::vector byte_id_; // byte value -> token id + std::unordered_map id_byte_value_; // token id -> byte value + std::unordered_map added_tokens_; + std::unordered_set added_first_bytes_; + std::vector added_lengths_; // distinct, descending + int unk_id_ = 0; +}; + + +// Versioned metadata serialized with `io/fields.h`. +struct BpeTokenizerBlob : public IFields { + const char* Name() const override { return "BpeTokenizerBlob"; } + + void VisitFields(IFieldsVisitor& visitor) override { + visitor(flags); + visitor(unk_id); + visitor(vocab); + visitor(merges); + visitor(added_ids); + } + + uint32_t flags = 0; + uint32_t unk_id = 0; + std::vector vocab; + std::vector merges; // left, right flattened + std::vector added_ids; +}; + +// Number of bytes in the UTF-8 sequence starting with lead byte `c`. +size_t Utf8Len(unsigned char c) { + if (c < 0x80) return 1; + if ((c >> 5) == 0x06) return 2; + if ((c >> 4) == 0x0e) return 3; + if ((c >> 3) == 0x1e) return 4; + return 1; // invalid lead byte: treat as a single byte +} + +uint64_t PairKey(int left, int right) { + return (static_cast(static_cast(left)) << 32) | + static_cast(right); +} + +// A candidate merge popped from the priority queue: lower `rank` merges first, +// ties broken by leftmost position, matching HuggingFace's BPE merge order. +struct QueuedMerge { + int rank; + int left; // symbol index of the pair's left element + bool operator>(const QueuedMerge& other) const { + if (rank != other.rank) return rank > other.rank; + return left > other.left; + } +}; + +// Decodes a run of byte-fallback bytes as UTF-8, appending to `out` (invalid +// sequences become U+FFFD, matching HuggingFace ByteFallback). +void FlushBytes(std::string* bytes, std::string* out) { + if (bytes->empty()) return; + size_t i = 0; + const std::string& b = *bytes; + while (i < b.size()) { + const size_t clen = Utf8Len(static_cast(b[i])); + if (i + clen <= b.size()) { + out->append(b, i, clen); + i += clen; + } else { + out->append("\xef\xbf\xbd"); // U+FFFD + ++i; + } + } + bytes->clear(); +} + +// Appends `token` to `out`, replacing U+2581 with a space (HF Replace decoder). +void AppendReplaced(const std::string& token, std::string* out) { + size_t pos = 0; + while (pos < token.size()) { + if (token.compare(pos, 3, kSpaceRepl) == 0) { + out->push_back(' '); + pos += 3; + } else { + out->push_back(token[pos]); + ++pos; + } + } +} + + +// Emits the tokenizer in compact IFields binary format. +// Serialization is delegated to IFieldsVisitor which walks BpeTokenizerBlob. +// +// The logical layout (packed into u32 words) is: +// [ flags: u32 ] +// [ unk_id: u32 ] +// [ vocab: vector ] +// [ merges: vector ] (flattened left_id, right_id order) +// [ added_ids: vector ] +// +// Native-endian. Strings pad up to u32 boundaries under IFields standards. +std::string BpeTokenizer::Serialize() const { + // Bulk: merges as (left_id, right_id) in ascending rank order. + std::vector>> by_rank; + by_rank.reserve(merge_ranks_.size()); + for (const auto& [key, rule] : merge_ranks_) { + by_rank.push_back( + {rule.rank, + {static_cast(key >> 32), static_cast(key & 0xffffffffu)}}); + } + std::sort(by_rank.begin(), by_rank.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + + // Bulk: added-token ids. + std::vector added_ids; + added_ids.reserve(added_tokens_.size()); + for (const auto& [content, id] : added_tokens_) added_ids.push_back(id); + std::sort(added_ids.begin(), added_ids.end()); + + BpeTokenizerBlob blob; + blob.unk_id = static_cast(unk_id_); + blob.vocab = id_to_token_; + blob.flags = kFlagSpaceReplace; + for (int b : byte_id_) { + if (b >= 0) { + blob.flags |= kFlagByteFallback; + break; + } + } + + blob.merges.reserve(by_rank.size() * 2); + for (const auto& e : by_rank) { + blob.merges.push_back(static_cast(e.second.first)); + blob.merges.push_back(static_cast(e.second.second)); + } + + blob.added_ids.reserve(added_ids.size()); + for (int id : added_ids) { + blob.added_ids.push_back(static_cast(id)); + } + + std::vector words = blob.Write(); + HWY_ASSERT(!words.empty()); + + return std::string(reinterpret_cast(words.data()), + words.size() * sizeof(uint32_t)); +} + +bool BpeTokenizer::Encode(std::string_view input, + std::vector& ids) const { + ids.clear(); + SplitOnAddedTokens(input, &ids); + return true; +} + +bool BpeTokenizer::Decode(hwy::Span ids, + std::string& detokenized) const { + detokenized.clear(); + std::string pending_bytes; // accumulates consecutive byte-fallback tokens + for (int id : ids) { + const int byte_value = IdByteValue(id); + if (byte_value >= 0) { + pending_bytes.push_back(static_cast(byte_value)); + continue; + } + FlushBytes(&pending_bytes, &detokenized); + AppendReplaced(IdToToken(id), &detokenized); + } + FlushBytes(&pending_bytes, &detokenized); + return true; +} + + +// Records the id and byte value of each "<0xHH>" byte-fallback token. +void BpeTokenizer::LoadByteTokens() { + byte_id_.assign(256, -1); + for (int b = 0; b < 256; ++b) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "<0x%02X>", b); + const auto it = vocab_.find(buf); + if (it != vocab_.end()) { + byte_id_[b] = it->second; + id_byte_value_[it->second] = b; + } + } +} + +// Inverse of Serialize. Reads the IFields blob (io/fields.h) then the +// raw byte region it describes. Aborts on malformed/truncated input. +bool BpeTokenizer::Deserialize(std::string_view data) { + if (data.size() % sizeof(uint32_t) != 0) { + HWY_WARN("Compact BPE blob size %zu not a multiple of 4.", data.size()); + return false; + } + std::vector words(data.size() / sizeof(uint32_t)); + if (!words.empty()) { + std::memcpy(words.data(), data.data(), data.size()); + } + + BpeTokenizerBlob blob; + const IFields::ReadResult res = + blob.Read(SerializedSpan(words.data(), words.size()), 0); + if (res.pos == 0) { + HWY_WARN("Compact BPE blob is invalid."); + return false; + } + + if (res.extra_u32) { + HWY_WARN( + "Compact BPE blob has %u extra fields; consider updating gemma.cpp.", + res.extra_u32); + } + unk_id_ = static_cast(blob.unk_id); + + id_to_token_ = std::move(blob.vocab); + const uint32_t vocab_count = static_cast(id_to_token_.size()); + + vocab_.clear(); + vocab_.reserve(static_cast(vocab_count) * 2); + for (uint32_t id = 0; id < vocab_count; ++id) { + const std::string& tok = id_to_token_[id]; + if (!tok.empty()) vocab_[tok] = static_cast(id); + } + + const uint32_t merges_count = static_cast(blob.merges.size() / 2); + merge_ranks_.clear(); + merge_ranks_.reserve(static_cast(merges_count) * 2); + for (uint32_t rank = 0; rank < merges_count; ++rank) { + const int left = static_cast(blob.merges[2 * rank]); + const int right = static_cast(blob.merges[2 * rank + 1]); + const auto mi = vocab_.find(IdToToken(left) + IdToToken(right)); + if (mi == vocab_.end()) continue; + merge_ranks_[PairKey(left, right)] = + MergeRule{static_cast(rank), mi->second}; + } + + const uint32_t added_count = static_cast(blob.added_ids.size()); + added_tokens_.clear(); + added_first_bytes_.clear(); + std::unordered_set lengths; + for (uint32_t i = 0; i < added_count; ++i) { + const std::string& content = + IdToToken(static_cast(blob.added_ids[i])); + if (content.empty()) continue; + added_tokens_[content] = static_cast(blob.added_ids[i]); + added_first_bytes_.insert(static_cast(content[0])); + lengths.insert(content.size()); + } + added_lengths_.assign(lengths.begin(), lengths.end()); + std::sort(added_lengths_.begin(), added_lengths_.end(), std::greater<>()); + + LoadByteTokens(); + return true; +} + +const std::string& BpeTokenizer::IdToToken(int id) const { + static const std::string kEmpty; + if (id < 0 || id >= static_cast(id_to_token_.size())) return kEmpty; + return id_to_token_[id]; +} + +int BpeTokenizer::IdByteValue(int id) const { + const auto it = id_byte_value_.find(id); + return it == id_byte_value_.end() ? -1 : it->second; +} + +// Returns the matched added-token id (and advances `i`) if one starts at +// `input[i]`, otherwise -1. +int BpeTokenizer::MatchAddedToken(std::string_view input, size_t i, + size_t* len) const { + if (added_first_bytes_.find(static_cast(input[i])) == + added_first_bytes_.end()) { + return -1; + } + for (size_t l : added_lengths_) { + if (i + l > input.size()) continue; + const auto it = added_tokens_.find(std::string(input.substr(i, l))); + if (it != added_tokens_.end()) { + *len = l; + return it->second; + } + } + return -1; +} + +// Carves `input` into added-token ids and normal-text spans, BPE-encoding the +// latter. Appends all resulting ids to `ids`. +void BpeTokenizer::SplitOnAddedTokens(std::string_view input, + std::vector* ids) const { + std::string buffer; + size_t i = 0; + while (i < input.size()) { + size_t len = 0; + const int added = MatchAddedToken(input, i, &len); + if (added >= 0) { + EncodeSpan(buffer, ids); + buffer.clear(); + ids->push_back(added); + i += len; + } else { + buffer.push_back(input[i]); + ++i; + } + } + EncodeSpan(buffer, ids); +} + +// Replaces ' ' with U+2581. +std::string BpeTokenizer::Normalize(std::string_view text) const { + std::string out; + out.reserve(text.size()); + for (char c : text) { + if (c == ' ') { + out += kSpaceRepl; + } else { + out.push_back(c); + } + } + return out; +} + +// Normalizes and BPE-encodes a normal-text span, appending ids. +void BpeTokenizer::EncodeSpan(std::string_view text, + std::vector* ids) const { + if (text.empty()) return; + const std::string norm = Normalize(text); + + // Initial symbols: one per whole-character vocab entry, else byte-fallback. + std::vector sym_id; + sym_id.reserve(norm.size()); + for (size_t i = 0; i < norm.size();) { + const size_t clen = Utf8Len(static_cast(norm[i])); + const std::string ch = norm.substr(i, clen); + const auto it = vocab_.find(std::string(ch)); + if (it != vocab_.end()) { + sym_id.push_back(it->second); + } else { + for (size_t b = 0; b < clen && i + b < norm.size(); ++b) { + const int bid = byte_id_[static_cast(norm[i + b])]; + sym_id.push_back(bid >= 0 ? bid : unk_id_); + } + } + i += clen; + } + + MergeSymbols(&sym_id); + ids->insert(ids->end(), sym_id.begin(), sym_id.end()); +} + +// In-place rank-priority BPE merge over a flat symbol list, using a doubly +// linked list with a lazily-cleaned min-heap of candidate merges. +void BpeTokenizer::MergeSymbols(std::vector* sym_id) const { + const int n = static_cast(sym_id->size()); + if (n < 2) return; + std::vector& id = *sym_id; + std::vector prev(n), next(n); + std::vector dead(n, 0); + for (int i = 0; i < n; ++i) { + prev[i] = i - 1; + next[i] = (i + 1 < n) ? i + 1 : -1; + } + + std::priority_queue, std::greater<>> + queue; + auto push_pair = [&](int left) { + if (left < 0 || next[left] < 0) return; + const auto it = merge_ranks_.find(PairKey(id[left], id[next[left]])); + if (it != merge_ranks_.end()) queue.push({it->second.rank, left}); + }; + for (int i = 0; i + 1 < n; ++i) push_pair(i); + + while (!queue.empty()) { + const QueuedMerge top = queue.top(); + queue.pop(); + const int left = top.left; + if (dead[left]) continue; + const int right = next[left]; + if (right < 0 || dead[right]) continue; + const auto it = merge_ranks_.find(PairKey(id[left], id[right])); + if (it == merge_ranks_.end() || it->second.rank != top.rank) continue; + + id[left] = it->second.new_id; + dead[right] = 1; + next[left] = next[right]; + if (next[left] >= 0) prev[next[left]] = left; + push_pair(prev[left]); + push_pair(left); + } + + std::vector out; + out.reserve(n); + int first = 0; + while (first >= 0 && dead[first]) first = next[first]; + for (int i = first; i >= 0; i = next[i]) out.push_back(id[i]); + *sym_id = std::move(out); +} + +} // namespace + +std::unique_ptr CreateBpeTokenizer(const std::string& blob) { + PROFILER_ZONE("Startup.tokenizer"); + auto bpe = std::make_unique(); + if (!bpe->Deserialize(blob)) { + HWY_ABORT("Failed to load BpeTokenizer from compact blob."); + } + return bpe; +} + +} // namespace gcpp diff --git a/tokenizer/bpe_tokenizer.h b/tokenizer/bpe_tokenizer.h new file mode 100644 index 00000000..51cd6617 --- /dev/null +++ b/tokenizer/bpe_tokenizer.h @@ -0,0 +1,30 @@ +// Copyright 2024 Google LLC +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_GEMMA_CPP_TOKENIZER_BPE_TOKENIZER_H_ +#define THIRD_PARTY_GEMMA_CPP_TOKENIZER_BPE_TOKENIZER_H_ + +#include +#include + +#include "tokenizer/tokenizer.h" + +namespace gcpp { + +std::unique_ptr CreateBpeTokenizer(const std::string& blob); + +} // namespace gcpp + +#endif // THIRD_PARTY_GEMMA_CPP_TOKENIZER_BPE_TOKENIZER_H_ diff --git a/tokenizer/bpe_tokenizer_test.cc b/tokenizer/bpe_tokenizer_test.cc new file mode 100644 index 00000000..1c091913 --- /dev/null +++ b/tokenizer/bpe_tokenizer_test.cc @@ -0,0 +1,194 @@ +// Copyright 2024 Google LLC +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "gemma/tokenizer.h" +#include "tokenizer/bpe_tokenizer.h" +#include "io/io.h" + +namespace gcpp { +namespace { + +// Tokenizer fixtures live next to the downloaded Gemma checkpoint. Tests run +// with the working directory set to the repository root. +constexpr const char* kTokenizerJson = + "tokenizer/testdata/tokenizer.json"; +constexpr const char* kTokenizerPacked = + "tokenizer/testdata/tokenizer.packed"; +constexpr const char* kTokenizerModel = + "tokenizer/testdata/tokenizer.model"; +constexpr const char* kCorpus = + "testdata/frankenstein.txt"; + +std::string ReadFileOrAbort(const std::string& path) { + return ReadFileToString(Path(path)); +} + +std::vector ReadLines(const std::string& path) { + std::string content = ReadFileToString(Path(path)); + std::vector lines; + std::stringstream ss(content); + std::string line; + while (std::getline(ss, line)) { + lines.push_back(line); + } + return lines; +} + +// Shares the tokenizer construction across test cases. +class BpeTokenizerTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { + sp_ = new GemmaTokenizer(ReadFileOrAbort(kTokenizerModel)); + bpe_ = new GemmaTokenizer( + CreateBpeTokenizer(ReadFileOrAbort(kTokenizerPacked))); + } + static void TearDownTestSuite() { + delete sp_; + delete bpe_; + sp_ = nullptr; + bpe_ = nullptr; + } + + // Asserts the BPE encoding of `text` matches SentencePiece exactly. + void ExpectSameIds(const std::string& text) { + std::vector sp_ids, bpe_ids; + ASSERT_TRUE(sp_->Encode(text, &sp_ids)); + ASSERT_TRUE(bpe_->Encode(text, &bpe_ids)); + EXPECT_EQ(bpe_ids, sp_ids) << "Mismatch on: [" << text << "]"; + } + + static inline GemmaTokenizer* sp_ = nullptr; + static inline GemmaTokenizer* bpe_ = nullptr; +}; + +TEST_F(BpeTokenizerTest, MatchesSentencePieceOnCorpus) { + const std::vector lines = ReadLines(kCorpus); + ASSERT_FALSE(lines.empty()); + size_t total_tokens = 0; + for (const std::string& line : lines) { + std::vector sp_ids, bpe_ids; + ASSERT_TRUE(sp_->Encode(line, &sp_ids)); + ASSERT_TRUE(bpe_->Encode(line, &bpe_ids)); + ASSERT_EQ(bpe_ids, sp_ids) << "Mismatch on line: [" << line << "]"; + total_tokens += sp_ids.size(); + } + EXPECT_GT(total_tokens, 0u); +} + +TEST_F(BpeTokenizerTest, MatchesOnWholeFile) { + ExpectSameIds(ReadFileOrAbort(kCorpus)); +} + +TEST_F(BpeTokenizerTest, EdgeCases) { + ExpectSameIds(""); + ExpectSameIds(" "); + ExpectSameIds(" "); + ExpectSameIds(" leading spaces"); + ExpectSameIds("trailing spaces "); + ExpectSameIds("\n"); + ExpectSameIds("\t\ttabs"); + ExpectSameIds("Hello, world!"); + ExpectSameIds("multiple internal spaces"); + ExpectSameIds("CamelCaseAndnumbers12345"); + ExpectSameIds("punctuation?!.,;:'\"()[]{}"); +} + +TEST_F(BpeTokenizerTest, Unicode) { + ExpectSameIds("café déjà vu naïve"); + ExpectSameIds("Ελληνικά κείμενο"); + ExpectSameIds("日本語のテキスト"); + ExpectSameIds("emoji 😀🎉🚀 fall back to bytes"); + ExpectSameIds("mixed 中文 and English 123"); +} + +TEST_F(BpeTokenizerTest, SpecialTokens) { + ExpectSameIds("user\n"); + ExpectSameIds("model\n"); + ExpectSameIds("\n"); + ExpectSameIds("helloworld"); + ExpectSameIds("\n\n"); + ExpectSameIds("\n\n"); +} + +TEST_F(BpeTokenizerTest, DecodeRoundTrip) { + const std::vector lines = ReadLines(kCorpus); + ASSERT_FALSE(lines.empty()); + for (const std::string& line : lines) { + std::vector ids; + ASSERT_TRUE(sp_->Encode(line, &ids)); + std::string bpe_text, sp_text; + ASSERT_TRUE(bpe_->Decode(ids, &bpe_text)); + ASSERT_TRUE(sp_->Decode(ids, &sp_text)); + EXPECT_EQ(bpe_text, sp_text) << "Decode mismatch on line: [" << line << "]"; + EXPECT_EQ(bpe_text, line) + << "Round-trip mismatch on line: [" << line << "]"; + } +} + +// A tokenizer rebuilt from the compact binary should behave identically to +// the one parsed from tokenizer.json. +TEST_F(BpeTokenizerTest, CompactRoundTrip) { + const std::string compact = bpe_->Serialize(); + EXPECT_LT(compact.size(), ReadFileOrAbort(kTokenizerJson).size()); + + const GemmaTokenizer compact_bpe(CreateBpeTokenizer(compact)); + + const std::vector samples = { + "", + " ", + "Hello, world!", + " leading spaces", + "trailing spaces ", + "café déjà vu naïve", + "日本語のテキスト", + "emoji 😀🎉🚀 fall back to bytes", + "user\n", + "helloworld", + "CamelCaseAndnumbers12345", + "punctuation?!.,;:'\"()[]{}", + }; + for (const std::string& s : samples) { + std::vector json_ids, compact_ids; + ASSERT_TRUE(bpe_->Encode(s, &json_ids)); + ASSERT_TRUE(compact_bpe.Encode(s, &compact_ids)); + EXPECT_EQ(compact_ids, json_ids) << "Mismatch on: [" << s << "]"; + } + + const std::vector lines = ReadLines(kCorpus); + ASSERT_FALSE(lines.empty()); + for (const std::string& line : lines) { + std::vector json_ids, compact_ids; + ASSERT_TRUE(bpe_->Encode(line, &json_ids)); + ASSERT_TRUE(compact_bpe.Encode(line, &compact_ids)); + ASSERT_EQ(compact_ids, json_ids) + << "Mismatch on corpus line: [" << line << "]"; + } + + std::vector ids; + ASSERT_TRUE(bpe_->Encode("Hello world", &ids)); + std::string json_text, compact_text; + ASSERT_TRUE(bpe_->Decode(ids, &json_text)); + ASSERT_TRUE(compact_bpe.Decode(ids, &compact_text)); + EXPECT_EQ(compact_text, json_text); +} + +} // namespace +} // namespace gcpp diff --git a/tokenizer/sentencepiece_tokenizer.cc b/tokenizer/sentencepiece_tokenizer.cc new file mode 100644 index 00000000..da938435 --- /dev/null +++ b/tokenizer/sentencepiece_tokenizer.cc @@ -0,0 +1,72 @@ +// Copyright 2024 Google LLC +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "tokenizer/tokenizer.h" +#include "hwy/aligned_allocator.h" // hwy::Span +#include "hwy/base.h" // HWY_ABORT +#include "hwy/profiler.h" +// copybara:import_next_line:sentencepiece +#include "src/sentencepiece_processor.h" +#include "tokenizer/sentencepiece_tokenizer.h" + +namespace gcpp { + +namespace { + +class SentencePieceTokenizer : public Tokenizer { + public: + SentencePieceTokenizer() = default; + + std::string Serialize() const override { + return spp_.serialized_model_proto(); + } + + bool Deserialize(std::string_view data) override { + PROFILER_ZONE("Startup.tokenizer"); + return spp_.LoadFromSerializedProto(data).ok(); + } + + bool Encode(std::string_view input, std::vector& ids) const override { + return spp_.Encode(input, &ids).ok(); + } + + bool Decode(hwy::Span ids, + std::string& detokenized) const override { + return spp_.Decode(std::vector(ids.begin(), ids.end()), + &detokenized).ok(); + } + + private: + sentencepiece::SentencePieceProcessor spp_; +}; + +} // namespace + +std::unique_ptr CreateSentencePieceTokenizer( + const std::string& tokenizer_proto) { + auto tokenizer = std::make_unique(); + if (!tokenizer->Deserialize(tokenizer_proto)) { + HWY_ABORT("Failed to load tokenizer from %zu byte serialized proto.", + tokenizer_proto.size()); + } + return tokenizer; +} + +} // namespace gcpp diff --git a/tokenizer/sentencepiece_tokenizer.h b/tokenizer/sentencepiece_tokenizer.h new file mode 100644 index 00000000..b1a1e752 --- /dev/null +++ b/tokenizer/sentencepiece_tokenizer.h @@ -0,0 +1,31 @@ +// Copyright 2024 Google LLC +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_GEMMA_CPP_TOKENIZER_SENTENCEPIECE_TOKENIZER_H_ +#define THIRD_PARTY_GEMMA_CPP_TOKENIZER_SENTENCEPIECE_TOKENIZER_H_ + +#include +#include + +#include "tokenizer/tokenizer.h" + +namespace gcpp { + +std::unique_ptr CreateSentencePieceTokenizer( + const std::string& tokenizer_proto); + +} // namespace gcpp + +#endif // THIRD_PARTY_GEMMA_CPP_TOKENIZER_SENTENCEPIECE_TOKENIZER_H_ diff --git a/tokenizer/testdata/tokenizer.json b/tokenizer/testdata/tokenizer.json new file mode 100644 index 00000000..298fb150 --- /dev/null +++ b/tokenizer/testdata/tokenizer.json @@ -0,0 +1,2379611 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 5, + "content": "[multimodal]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 6, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 7, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 8, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 9, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 10, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 11, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 12, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 13, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 14, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 15, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 16, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 17, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 18, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 19, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 20, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 21, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 22, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 23, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 24, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 25, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 26, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 27, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 28, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 29, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 30, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 31, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 32, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 33, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 34, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 35, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 36, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 37, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 38, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 39, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 40, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 41, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 42, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 43, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 44, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 45, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 46, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 47, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 48, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 49, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 50, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 51, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 52, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 53, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 54, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 55, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 56, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 57, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 58, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 59, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 60, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 61, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 62, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 63, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 64, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 65, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 66, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 67, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 68, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 69, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 70, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 71, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 72, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 73, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 74, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 75, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 76, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 77, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 78, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 79, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 80, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 81, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 82, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 83, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 84, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 85, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 86, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 87, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 88, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 89, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 90, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 91, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 92, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 93, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 94, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 95, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 96, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 97, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 98, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 99, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 100, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 101, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 102, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 103, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 104, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 105, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 106, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 107, + "content": "\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 108, + "content": "\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 109, + "content": "\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 110, + "content": "\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 111, + "content": "\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 112, + "content": "\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 113, + "content": "\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 114, + "content": "\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 115, + "content": "\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 116, + "content": "\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 117, + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 118, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 119, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 120, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 121, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 122, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 123, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 124, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 125, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 126, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 127, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 128, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 129, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 130, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 131, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 132, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 133, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 134, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 135, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 136, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 137, + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 138, + "content": "▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 139, + "content": "▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 140, + "content": "▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 141, + "content": "▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 142, + "content": "▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 143, + "content": "▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 144, + "content": "▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 145, + "content": "▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 146, + "content": "▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 147, + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 148, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 149, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 150, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 151, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 152, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 153, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 154, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 155, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 156, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 157, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 158, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 159, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 160, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 161, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 162, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 163, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 164, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 165, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 166, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 167, + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 168, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 169, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 171, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 172, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 173, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 174, + "content": "
", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 170, + "content": "
", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 175, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 176, + "content": "
", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 177, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 178, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 179, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 180, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 181, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 182, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 183, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 184, + "content": "

", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 185, + "content": "

", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 186, + "content": "

", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 187, + "content": "

", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 188, + "content": "

", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 189, + "content": "
", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 190, + "content": "
", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 191, + "content": "
", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 192, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 193, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 194, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 195, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 196, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 197, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 198, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 199, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 200, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 201, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 202, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 203, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 204, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 205, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 206, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 207, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 208, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 209, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 210, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 211, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 212, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 213, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 214, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 215, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 216, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 217, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 218, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 219, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 220, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 221, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 222, + "content": "