123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497 |
- // Copyright 2016 Google Inc.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.!
- // Code generated by protoc-gen-go. DO NOT EDIT.
- // versions:
- // protoc-gen-go v1.32.0
- // protoc v4.25.2
- // source: sentencepiece_model.proto
- package sentencepiece
- import (
- protoreflect "google.golang.org/protobuf/reflect/protoreflect"
- protoimpl "google.golang.org/protobuf/runtime/protoimpl"
- reflect "reflect"
- sync "sync"
- )
- const (
- // Verify that this generated code is sufficiently up-to-date.
- _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
- // Verify that runtime/protoimpl is sufficiently up-to-date.
- _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
- )
- // Model type. only have UNIGRAM now.
- type TrainerSpec_ModelType int32
- const (
- TrainerSpec_UNIGRAM TrainerSpec_ModelType = 1 // Unigram language model with dynamic algorithm
- TrainerSpec_BPE TrainerSpec_ModelType = 2 // Byte Pair Encoding
- TrainerSpec_WORD TrainerSpec_ModelType = 3 // Delimitered by whitespace.
- TrainerSpec_CHAR TrainerSpec_ModelType = 4 // tokenizes into character sequence
- )
- // Enum value maps for TrainerSpec_ModelType.
- var (
- TrainerSpec_ModelType_name = map[int32]string{
- 1: "UNIGRAM",
- 2: "BPE",
- 3: "WORD",
- 4: "CHAR",
- }
- TrainerSpec_ModelType_value = map[string]int32{
- "UNIGRAM": 1,
- "BPE": 2,
- "WORD": 3,
- "CHAR": 4,
- }
- )
- func (x TrainerSpec_ModelType) Enum() *TrainerSpec_ModelType {
- p := new(TrainerSpec_ModelType)
- *p = x
- return p
- }
- func (x TrainerSpec_ModelType) String() string {
- return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
- }
- func (TrainerSpec_ModelType) Descriptor() protoreflect.EnumDescriptor {
- return file_sentencepiece_model_proto_enumTypes[0].Descriptor()
- }
- func (TrainerSpec_ModelType) Type() protoreflect.EnumType {
- return &file_sentencepiece_model_proto_enumTypes[0]
- }
- func (x TrainerSpec_ModelType) Number() protoreflect.EnumNumber {
- return protoreflect.EnumNumber(x)
- }
- // Deprecated: Do not use.
- func (x *TrainerSpec_ModelType) UnmarshalJSON(b []byte) error {
- num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b)
- if err != nil {
- return err
- }
- *x = TrainerSpec_ModelType(num)
- return nil
- }
- // Deprecated: Use TrainerSpec_ModelType.Descriptor instead.
- func (TrainerSpec_ModelType) EnumDescriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{0, 0}
- }
- type ModelProto_SentencePiece_Type int32
- const (
- ModelProto_SentencePiece_NORMAL ModelProto_SentencePiece_Type = 1 // normal symbol
- ModelProto_SentencePiece_UNKNOWN ModelProto_SentencePiece_Type = 2 // unknown symbol. only <unk> for now.
- ModelProto_SentencePiece_CONTROL ModelProto_SentencePiece_Type = 3 // control symbols. </s>, <s>, <2ja> etc.
- ModelProto_SentencePiece_USER_DEFINED ModelProto_SentencePiece_Type = 4 // user defined symbols.
- // Typical usage of USER_DEFINED symbol
- // is placeholder.
- ModelProto_SentencePiece_BYTE ModelProto_SentencePiece_Type = 6 // byte symbols. Used when `byte_fallback` is true.
- ModelProto_SentencePiece_UNUSED ModelProto_SentencePiece_Type = 5 // this piece is not used.
- )
- // Enum value maps for ModelProto_SentencePiece_Type.
- var (
- ModelProto_SentencePiece_Type_name = map[int32]string{
- 1: "NORMAL",
- 2: "UNKNOWN",
- 3: "CONTROL",
- 4: "USER_DEFINED",
- 6: "BYTE",
- 5: "UNUSED",
- }
- ModelProto_SentencePiece_Type_value = map[string]int32{
- "NORMAL": 1,
- "UNKNOWN": 2,
- "CONTROL": 3,
- "USER_DEFINED": 4,
- "BYTE": 6,
- "UNUSED": 5,
- }
- )
- func (x ModelProto_SentencePiece_Type) Enum() *ModelProto_SentencePiece_Type {
- p := new(ModelProto_SentencePiece_Type)
- *p = x
- return p
- }
- func (x ModelProto_SentencePiece_Type) String() string {
- return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
- }
- func (ModelProto_SentencePiece_Type) Descriptor() protoreflect.EnumDescriptor {
- return file_sentencepiece_model_proto_enumTypes[1].Descriptor()
- }
- func (ModelProto_SentencePiece_Type) Type() protoreflect.EnumType {
- return &file_sentencepiece_model_proto_enumTypes[1]
- }
- func (x ModelProto_SentencePiece_Type) Number() protoreflect.EnumNumber {
- return protoreflect.EnumNumber(x)
- }
- // Deprecated: Do not use.
- func (x *ModelProto_SentencePiece_Type) UnmarshalJSON(b []byte) error {
- num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b)
- if err != nil {
- return err
- }
- *x = ModelProto_SentencePiece_Type(num)
- return nil
- }
- // Deprecated: Use ModelProto_SentencePiece_Type.Descriptor instead.
- func (ModelProto_SentencePiece_Type) EnumDescriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{3, 0, 0}
- }
- // TrainerSpec encodes a various parameters for SentencePiece training.
- // Next id: 55
- type TrainerSpec struct {
- state protoimpl.MessageState
- sizeCache protoimpl.SizeCache
- unknownFields protoimpl.UnknownFields
- extensionFields protoimpl.ExtensionFields
- // /////////////////////////////////////////////////////////////////
- // General parameters
- //
- // Input corpus files.
- //
- // Trainer accepts the following two formats:
- // A) Monolingual: plain text, one sentence per line.
- // B) Bilingual: TSV, source sentence <tab> target sentence
- // When bilingual data is passed, shared vocabulary model is built.
- // Note that the input file must be raw corpus, not a preprocessed corpus.
- // Trainer only loads the first `input_sentence_size` sentences specified
- // with this parameter.
- Input []string `protobuf:"bytes,1,rep,name=input" json:"input,omitempty"`
- // Input corpus format:
- // "text": one-sentence-per-line text format (default)
- // "tsv": sentence <tab> freq
- InputFormat *string `protobuf:"bytes,7,opt,name=input_format,json=inputFormat" json:"input_format,omitempty"`
- // Output model file prefix.
- // <model_prefix>.model and <model_prefix>.vocab are generated.
- ModelPrefix *string `protobuf:"bytes,2,opt,name=model_prefix,json=modelPrefix" json:"model_prefix,omitempty"`
- ModelType *TrainerSpec_ModelType `protobuf:"varint,3,opt,name=model_type,json=modelType,enum=sentencepiece.TrainerSpec_ModelType,def=1" json:"model_type,omitempty"`
- // Vocabulary size. 8k is the default size.
- VocabSize *int32 `protobuf:"varint,4,opt,name=vocab_size,json=vocabSize,def=8000" json:"vocab_size,omitempty"`
- // List of the languages this model can accept.
- // Since the model is language-agnostic, this field is used as a reference.
- AcceptLanguage []string `protobuf:"bytes,5,rep,name=accept_language,json=acceptLanguage" json:"accept_language,omitempty"`
- // Size of self-test samples, which are encoded in the model file.
- SelfTestSampleSize *int32 `protobuf:"varint,6,opt,name=self_test_sample_size,json=selfTestSampleSize,def=0" json:"self_test_sample_size,omitempty"`
- // Whether to use DP version of sentencepiece. Use it with TSV input format
- // (requires precomputed word tab counts to work).
- EnableDifferentialPrivacy *bool `protobuf:"varint,50,opt,name=enable_differential_privacy,json=enableDifferentialPrivacy,def=0" json:"enable_differential_privacy,omitempty"`
- // Set these parameters if you need DP version of sentencepiece.
- // std of noise to add.
- DifferentialPrivacyNoiseLevel *float32 `protobuf:"fixed32,51,opt,name=differential_privacy_noise_level,json=differentialPrivacyNoiseLevel,def=0" json:"differential_privacy_noise_level,omitempty"`
- // Clipping threshold to apply after adding noise. All the words with
- // frequency less than this value are dropped.
- DifferentialPrivacyClippingThreshold *uint64 `protobuf:"varint,52,opt,name=differential_privacy_clipping_threshold,json=differentialPrivacyClippingThreshold,def=0" json:"differential_privacy_clipping_threshold,omitempty"`
- // /////////////////////////////////////////////////////////////////
- // Training parameters.
- //
- // Uses characters which cover the corpus with the ratio of `chars_coverage`.
- // This parameter determines the set of basic Alphabet of sentence piece.
- // 1.0 - `chars_coverage` characters are treated as UNK.
- // See also required_chars field.
- CharacterCoverage *float32 `protobuf:"fixed32,10,opt,name=character_coverage,json=characterCoverage,def=0.9995" json:"character_coverage,omitempty"`
- // Maximum size of sentences the trainer loads from `input` parameter.
- // Trainer simply loads the `input` files in sequence.
- // It is better to shuffle the input corpus randomly.
- InputSentenceSize *uint64 `protobuf:"varint,11,opt,name=input_sentence_size,json=inputSentenceSize,def=0" json:"input_sentence_size,omitempty"`
- ShuffleInputSentence *bool `protobuf:"varint,19,opt,name=shuffle_input_sentence,json=shuffleInputSentence,def=1" json:"shuffle_input_sentence,omitempty"`
- // Maximum size of sentences to make seed sentence pieces.
- // Extended suffix array is constructed to extract frequent
- // sub-strings from the corpus. This uses 20N working space,
- // where N is the size of corpus.
- //
- // Deprecated: Marked as deprecated in sentencepiece_model.proto.
- MiningSentenceSize *int32 `protobuf:"varint,12,opt,name=mining_sentence_size,json=miningSentenceSize" json:"mining_sentence_size,omitempty"`
- // Maximum size of sentences to train sentence pieces.
- //
- // Deprecated: Marked as deprecated in sentencepiece_model.proto.
- TrainingSentenceSize *int32 `protobuf:"varint,13,opt,name=training_sentence_size,json=trainingSentenceSize" json:"training_sentence_size,omitempty"`
- // The size of seed sentencepieces.
- // `seed_sentencepiece_size` must be larger than `vocab_size`.
- SeedSentencepieceSize *int32 `protobuf:"varint,14,opt,name=seed_sentencepiece_size,json=seedSentencepieceSize,def=1000000" json:"seed_sentencepiece_size,omitempty"`
- // In every EM sub-iterations, keeps top
- // `shrinking_factor` * `current sentencepieces size` with respect to
- // the loss of the sentence piece. This value should be smaller than 1.0.
- ShrinkingFactor *float32 `protobuf:"fixed32,15,opt,name=shrinking_factor,json=shrinkingFactor,def=0.75" json:"shrinking_factor,omitempty"`
- // The maximum sentence length in byte. The sentences with the length
- // larger than `max_sentence_length` is simply ignored.
- // Longer input tends to bring the following risks:
- // - Overflow during EM training (unigram language model only)
- // - Performance drop because of O(n log n) cost in BPE.
- MaxSentenceLength *int32 `protobuf:"varint,18,opt,name=max_sentence_length,json=maxSentenceLength,def=4192" json:"max_sentence_length,omitempty"`
- // Number of threads in the training.
- NumThreads *int32 `protobuf:"varint,16,opt,name=num_threads,json=numThreads,def=16" json:"num_threads,omitempty"`
- // Number of EM sub iterations.
- NumSubIterations *int32 `protobuf:"varint,17,opt,name=num_sub_iterations,json=numSubIterations,def=2" json:"num_sub_iterations,omitempty"`
- // /////////////////////////////////////////////////////////////////
- // SentencePiece parameters which control the shapes of sentence piece.
- //
- // Maximum length of sentencepiece.
- MaxSentencepieceLength *int32 `protobuf:"varint,20,opt,name=max_sentencepiece_length,json=maxSentencepieceLength,def=16" json:"max_sentencepiece_length,omitempty"`
- // Uses Unicode script to split sentence pieces.
- // When `split_by_unicode_script` is true, we do not allow sentence piece to
- // include multiple Unicode scripts, e.g. "F1" is not a valid piece.
- // Exception: CJ characters (Hiragana/Katakana/Han) are all handled
- // as one script type, since Japanese word can consist of multiple scripts.
- // This exception is always applied regardless of the accept-language
- // parameter.
- SplitByUnicodeScript *bool `protobuf:"varint,21,opt,name=split_by_unicode_script,json=splitByUnicodeScript,def=1" json:"split_by_unicode_script,omitempty"`
- // When `split_by_number` is true, put a boundary between number and
- // non-number transition. If we want to treat "F1" is one token, set this flag
- // to be false.
- SplitByNumber *bool `protobuf:"varint,23,opt,name=split_by_number,json=splitByNumber,def=1" json:"split_by_number,omitempty"`
- // Use a white space to split sentence pieces.
- // When `split_by_whitespace` is false, we may have the piece containing
- // a white space in the middle. e.g., "in_the".
- SplitByWhitespace *bool `protobuf:"varint,22,opt,name=split_by_whitespace,json=splitByWhitespace,def=1" json:"split_by_whitespace,omitempty"`
- // Adds whitespace symbol (_) as a suffix instead of prefix. e.g., _hello =>
- // hello_. When `treat_whitespace_as_suffix` is true,
- // NormalizerSpec::add_dummy_prefix will add the dummy whitespace to the end
- // of sentence.
- TreatWhitespaceAsSuffix *bool `protobuf:"varint,24,opt,name=treat_whitespace_as_suffix,json=treatWhitespaceAsSuffix,def=0" json:"treat_whitespace_as_suffix,omitempty"`
- // Allows pieces that only contain whitespaces instead of appearing only as
- // prefix or suffix of other pieces.
- AllowWhitespaceOnlyPieces *bool `protobuf:"varint,26,opt,name=allow_whitespace_only_pieces,json=allowWhitespaceOnlyPieces,def=0" json:"allow_whitespace_only_pieces,omitempty"`
- // Split all digits (0-9) into separate pieces.
- SplitDigits *bool `protobuf:"varint,25,opt,name=split_digits,json=splitDigits,def=0" json:"split_digits,omitempty"`
- // Defines the pre-tokenization delimiter.
- // When specified, no pieces crossing this delimiter is not included
- // in the vocab. Then the delimiter string is virtually ignored
- // during the training. This field can allows constraints on the vocabulary
- // selection. Note that this field is available on unigram mode.
- PretokenizationDelimiter *string `protobuf:"bytes,53,opt,name=pretokenization_delimiter,json=pretokenizationDelimiter,def=" json:"pretokenization_delimiter,omitempty"`
- // /////////////////////////////////////////////////////////////////
- // Vocabulary management
- //
- // Defines control symbols used as an indicator to
- // change the behavior of the decoder. <s> and </s> are pre-defined.
- // We can use this field to encode various meta information,
- // including language indicator in multilingual model.
- // These symbols are not visible to users, but visible to
- // the decoder. Note that when the input sentence contains control symbols,
- // they are not treated as one token, but segmented into normal pieces.
- // Control symbols must be inserted independently from the segmentation.
- ControlSymbols []string `protobuf:"bytes,30,rep,name=control_symbols,json=controlSymbols" json:"control_symbols,omitempty"`
- // Defines user defined symbols.
- // These symbols are added with extremely high score
- // so they are always treated as one unique symbol in any context.
- // Typical usage of user_defined_symbols is placeholder for named entities.
- UserDefinedSymbols []string `protobuf:"bytes,31,rep,name=user_defined_symbols,json=userDefinedSymbols" json:"user_defined_symbols,omitempty"`
- // Defines required characters. Each UTF8 character in this string is included
- // in the character set regardless of character_coverage value. Unlike
- // user_defined_symbols, these characters have scores based on the frequency
- // on input sentences, and the model can form subwords using characters
- // in this field.
- RequiredChars *string `protobuf:"bytes,36,opt,name=required_chars,json=requiredChars" json:"required_chars,omitempty"`
- // Decomposes unknown pieces into UTF-8 bytes.
- ByteFallback *bool `protobuf:"varint,35,opt,name=byte_fallback,json=byteFallback,def=0" json:"byte_fallback,omitempty"`
- // When creating the vocabulary file, defines whether or not to additionally
- // output the score for each piece.
- VocabularyOutputPieceScore *bool `protobuf:"varint,32,opt,name=vocabulary_output_piece_score,json=vocabularyOutputPieceScore,def=1" json:"vocabulary_output_piece_score,omitempty"`
- // `vocab_size` is treated as hard limit. Crash if
- // the model can not produce the vocab of size `vocab_size`,
- // When `hard_vocab_limit` is false, vocab_size is treated
- // as soft limit. Note that when model_type=char,
- // always assumes hard_vocab_limit = false.
- HardVocabLimit *bool `protobuf:"varint,33,opt,name=hard_vocab_limit,json=hardVocabLimit,def=1" json:"hard_vocab_limit,omitempty"`
- // use all symbols for vocab extraction. This flag is valid
- // if model type is either CHAR or WORD
- UseAllVocab *bool `protobuf:"varint,34,opt,name=use_all_vocab,json=useAllVocab,def=0" json:"use_all_vocab,omitempty"`
- // /////////////////////////////////////////////////////////////////
- // Reserved special meta tokens.
- // * -1 is not used.
- // * unk_id must not be -1.
- // Id must start with 0 and be contiguous.
- UnkId *int32 `protobuf:"varint,40,opt,name=unk_id,json=unkId,def=0" json:"unk_id,omitempty"` // <unk>
- BosId *int32 `protobuf:"varint,41,opt,name=bos_id,json=bosId,def=1" json:"bos_id,omitempty"` // <s>
- EosId *int32 `protobuf:"varint,42,opt,name=eos_id,json=eosId,def=2" json:"eos_id,omitempty"` // </s>
- PadId *int32 `protobuf:"varint,43,opt,name=pad_id,json=padId,def=-1" json:"pad_id,omitempty"` // <pad> (padding)
- UnkPiece *string `protobuf:"bytes,45,opt,name=unk_piece,json=unkPiece,def=<unk>" json:"unk_piece,omitempty"`
- BosPiece *string `protobuf:"bytes,46,opt,name=bos_piece,json=bosPiece,def=<s>" json:"bos_piece,omitempty"`
- EosPiece *string `protobuf:"bytes,47,opt,name=eos_piece,json=eosPiece,def=</s>" json:"eos_piece,omitempty"`
- PadPiece *string `protobuf:"bytes,48,opt,name=pad_piece,json=padPiece,def=<pad>" json:"pad_piece,omitempty"`
- // Encodes <unk> into U+2047 (DOUBLE QUESTION MARK),
- // since this character can be useful both for user and
- // developer. We can easily figure out that <unk> is emitted.
- UnkSurface *string `protobuf:"bytes,44,opt,name=unk_surface,json=unkSurface,def= ⁇ " json:"unk_surface,omitempty"`
- // Increase bit depth to allow unigram model training on large
- // (>10M sentences) corpora. A Side-effect of enabling this flag
- // is increased memory usage.
- TrainExtremelyLargeCorpus *bool `protobuf:"varint,49,opt,name=train_extremely_large_corpus,json=trainExtremelyLargeCorpus,def=0" json:"train_extremely_large_corpus,omitempty"`
- // Path to a seed sentencepieces file, with one tab-separated
- // seed sentencepiece <tab> frequency per line.
- SeedSentencepiecesFile *string `protobuf:"bytes,54,opt,name=seed_sentencepieces_file,json=seedSentencepiecesFile,def=" json:"seed_sentencepieces_file,omitempty"`
- }
- // Default values for TrainerSpec fields.
- const (
- Default_TrainerSpec_ModelType = TrainerSpec_UNIGRAM
- Default_TrainerSpec_VocabSize = int32(8000)
- Default_TrainerSpec_SelfTestSampleSize = int32(0)
- Default_TrainerSpec_EnableDifferentialPrivacy = bool(false)
- Default_TrainerSpec_DifferentialPrivacyNoiseLevel = float32(0)
- Default_TrainerSpec_DifferentialPrivacyClippingThreshold = uint64(0)
- Default_TrainerSpec_CharacterCoverage = float32(0.9994999766349792)
- Default_TrainerSpec_InputSentenceSize = uint64(0)
- Default_TrainerSpec_ShuffleInputSentence = bool(true)
- Default_TrainerSpec_SeedSentencepieceSize = int32(1000000)
- Default_TrainerSpec_ShrinkingFactor = float32(0.75)
- Default_TrainerSpec_MaxSentenceLength = int32(4192)
- Default_TrainerSpec_NumThreads = int32(16)
- Default_TrainerSpec_NumSubIterations = int32(2)
- Default_TrainerSpec_MaxSentencepieceLength = int32(16)
- Default_TrainerSpec_SplitByUnicodeScript = bool(true)
- Default_TrainerSpec_SplitByNumber = bool(true)
- Default_TrainerSpec_SplitByWhitespace = bool(true)
- Default_TrainerSpec_TreatWhitespaceAsSuffix = bool(false)
- Default_TrainerSpec_AllowWhitespaceOnlyPieces = bool(false)
- Default_TrainerSpec_SplitDigits = bool(false)
- Default_TrainerSpec_PretokenizationDelimiter = string("")
- Default_TrainerSpec_ByteFallback = bool(false)
- Default_TrainerSpec_VocabularyOutputPieceScore = bool(true)
- Default_TrainerSpec_HardVocabLimit = bool(true)
- Default_TrainerSpec_UseAllVocab = bool(false)
- Default_TrainerSpec_UnkId = int32(0)
- Default_TrainerSpec_BosId = int32(1)
- Default_TrainerSpec_EosId = int32(2)
- Default_TrainerSpec_PadId = int32(-1)
- Default_TrainerSpec_UnkPiece = string("<unk>")
- Default_TrainerSpec_BosPiece = string("<s>")
- Default_TrainerSpec_EosPiece = string("</s>")
- Default_TrainerSpec_PadPiece = string("<pad>")
- Default_TrainerSpec_UnkSurface = string(" ⁇ ")
- Default_TrainerSpec_TrainExtremelyLargeCorpus = bool(false)
- Default_TrainerSpec_SeedSentencepiecesFile = string("")
- )
- func (x *TrainerSpec) Reset() {
- *x = TrainerSpec{}
- if protoimpl.UnsafeEnabled {
- mi := &file_sentencepiece_model_proto_msgTypes[0]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
- }
- }
- func (x *TrainerSpec) String() string {
- return protoimpl.X.MessageStringOf(x)
- }
- func (*TrainerSpec) ProtoMessage() {}
- func (x *TrainerSpec) ProtoReflect() protoreflect.Message {
- mi := &file_sentencepiece_model_proto_msgTypes[0]
- if protoimpl.UnsafeEnabled && x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
- }
- // Deprecated: Use TrainerSpec.ProtoReflect.Descriptor instead.
- func (*TrainerSpec) Descriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{0}
- }
- func (x *TrainerSpec) GetInput() []string {
- if x != nil {
- return x.Input
- }
- return nil
- }
- func (x *TrainerSpec) GetInputFormat() string {
- if x != nil && x.InputFormat != nil {
- return *x.InputFormat
- }
- return ""
- }
- func (x *TrainerSpec) GetModelPrefix() string {
- if x != nil && x.ModelPrefix != nil {
- return *x.ModelPrefix
- }
- return ""
- }
- func (x *TrainerSpec) GetModelType() TrainerSpec_ModelType {
- if x != nil && x.ModelType != nil {
- return *x.ModelType
- }
- return Default_TrainerSpec_ModelType
- }
- func (x *TrainerSpec) GetVocabSize() int32 {
- if x != nil && x.VocabSize != nil {
- return *x.VocabSize
- }
- return Default_TrainerSpec_VocabSize
- }
- func (x *TrainerSpec) GetAcceptLanguage() []string {
- if x != nil {
- return x.AcceptLanguage
- }
- return nil
- }
- func (x *TrainerSpec) GetSelfTestSampleSize() int32 {
- if x != nil && x.SelfTestSampleSize != nil {
- return *x.SelfTestSampleSize
- }
- return Default_TrainerSpec_SelfTestSampleSize
- }
- func (x *TrainerSpec) GetEnableDifferentialPrivacy() bool {
- if x != nil && x.EnableDifferentialPrivacy != nil {
- return *x.EnableDifferentialPrivacy
- }
- return Default_TrainerSpec_EnableDifferentialPrivacy
- }
- func (x *TrainerSpec) GetDifferentialPrivacyNoiseLevel() float32 {
- if x != nil && x.DifferentialPrivacyNoiseLevel != nil {
- return *x.DifferentialPrivacyNoiseLevel
- }
- return Default_TrainerSpec_DifferentialPrivacyNoiseLevel
- }
- func (x *TrainerSpec) GetDifferentialPrivacyClippingThreshold() uint64 {
- if x != nil && x.DifferentialPrivacyClippingThreshold != nil {
- return *x.DifferentialPrivacyClippingThreshold
- }
- return Default_TrainerSpec_DifferentialPrivacyClippingThreshold
- }
- func (x *TrainerSpec) GetCharacterCoverage() float32 {
- if x != nil && x.CharacterCoverage != nil {
- return *x.CharacterCoverage
- }
- return Default_TrainerSpec_CharacterCoverage
- }
- func (x *TrainerSpec) GetInputSentenceSize() uint64 {
- if x != nil && x.InputSentenceSize != nil {
- return *x.InputSentenceSize
- }
- return Default_TrainerSpec_InputSentenceSize
- }
- func (x *TrainerSpec) GetShuffleInputSentence() bool {
- if x != nil && x.ShuffleInputSentence != nil {
- return *x.ShuffleInputSentence
- }
- return Default_TrainerSpec_ShuffleInputSentence
- }
- // Deprecated: Marked as deprecated in sentencepiece_model.proto.
- func (x *TrainerSpec) GetMiningSentenceSize() int32 {
- if x != nil && x.MiningSentenceSize != nil {
- return *x.MiningSentenceSize
- }
- return 0
- }
- // Deprecated: Marked as deprecated in sentencepiece_model.proto.
- func (x *TrainerSpec) GetTrainingSentenceSize() int32 {
- if x != nil && x.TrainingSentenceSize != nil {
- return *x.TrainingSentenceSize
- }
- return 0
- }
- func (x *TrainerSpec) GetSeedSentencepieceSize() int32 {
- if x != nil && x.SeedSentencepieceSize != nil {
- return *x.SeedSentencepieceSize
- }
- return Default_TrainerSpec_SeedSentencepieceSize
- }
- func (x *TrainerSpec) GetShrinkingFactor() float32 {
- if x != nil && x.ShrinkingFactor != nil {
- return *x.ShrinkingFactor
- }
- return Default_TrainerSpec_ShrinkingFactor
- }
- func (x *TrainerSpec) GetMaxSentenceLength() int32 {
- if x != nil && x.MaxSentenceLength != nil {
- return *x.MaxSentenceLength
- }
- return Default_TrainerSpec_MaxSentenceLength
- }
- func (x *TrainerSpec) GetNumThreads() int32 {
- if x != nil && x.NumThreads != nil {
- return *x.NumThreads
- }
- return Default_TrainerSpec_NumThreads
- }
- func (x *TrainerSpec) GetNumSubIterations() int32 {
- if x != nil && x.NumSubIterations != nil {
- return *x.NumSubIterations
- }
- return Default_TrainerSpec_NumSubIterations
- }
- func (x *TrainerSpec) GetMaxSentencepieceLength() int32 {
- if x != nil && x.MaxSentencepieceLength != nil {
- return *x.MaxSentencepieceLength
- }
- return Default_TrainerSpec_MaxSentencepieceLength
- }
- func (x *TrainerSpec) GetSplitByUnicodeScript() bool {
- if x != nil && x.SplitByUnicodeScript != nil {
- return *x.SplitByUnicodeScript
- }
- return Default_TrainerSpec_SplitByUnicodeScript
- }
- func (x *TrainerSpec) GetSplitByNumber() bool {
- if x != nil && x.SplitByNumber != nil {
- return *x.SplitByNumber
- }
- return Default_TrainerSpec_SplitByNumber
- }
- func (x *TrainerSpec) GetSplitByWhitespace() bool {
- if x != nil && x.SplitByWhitespace != nil {
- return *x.SplitByWhitespace
- }
- return Default_TrainerSpec_SplitByWhitespace
- }
- func (x *TrainerSpec) GetTreatWhitespaceAsSuffix() bool {
- if x != nil && x.TreatWhitespaceAsSuffix != nil {
- return *x.TreatWhitespaceAsSuffix
- }
- return Default_TrainerSpec_TreatWhitespaceAsSuffix
- }
- func (x *TrainerSpec) GetAllowWhitespaceOnlyPieces() bool {
- if x != nil && x.AllowWhitespaceOnlyPieces != nil {
- return *x.AllowWhitespaceOnlyPieces
- }
- return Default_TrainerSpec_AllowWhitespaceOnlyPieces
- }
- func (x *TrainerSpec) GetSplitDigits() bool {
- if x != nil && x.SplitDigits != nil {
- return *x.SplitDigits
- }
- return Default_TrainerSpec_SplitDigits
- }
- func (x *TrainerSpec) GetPretokenizationDelimiter() string {
- if x != nil && x.PretokenizationDelimiter != nil {
- return *x.PretokenizationDelimiter
- }
- return Default_TrainerSpec_PretokenizationDelimiter
- }
- func (x *TrainerSpec) GetControlSymbols() []string {
- if x != nil {
- return x.ControlSymbols
- }
- return nil
- }
- func (x *TrainerSpec) GetUserDefinedSymbols() []string {
- if x != nil {
- return x.UserDefinedSymbols
- }
- return nil
- }
- func (x *TrainerSpec) GetRequiredChars() string {
- if x != nil && x.RequiredChars != nil {
- return *x.RequiredChars
- }
- return ""
- }
- func (x *TrainerSpec) GetByteFallback() bool {
- if x != nil && x.ByteFallback != nil {
- return *x.ByteFallback
- }
- return Default_TrainerSpec_ByteFallback
- }
- func (x *TrainerSpec) GetVocabularyOutputPieceScore() bool {
- if x != nil && x.VocabularyOutputPieceScore != nil {
- return *x.VocabularyOutputPieceScore
- }
- return Default_TrainerSpec_VocabularyOutputPieceScore
- }
- func (x *TrainerSpec) GetHardVocabLimit() bool {
- if x != nil && x.HardVocabLimit != nil {
- return *x.HardVocabLimit
- }
- return Default_TrainerSpec_HardVocabLimit
- }
- func (x *TrainerSpec) GetUseAllVocab() bool {
- if x != nil && x.UseAllVocab != nil {
- return *x.UseAllVocab
- }
- return Default_TrainerSpec_UseAllVocab
- }
- func (x *TrainerSpec) GetUnkId() int32 {
- if x != nil && x.UnkId != nil {
- return *x.UnkId
- }
- return Default_TrainerSpec_UnkId
- }
- func (x *TrainerSpec) GetBosId() int32 {
- if x != nil && x.BosId != nil {
- return *x.BosId
- }
- return Default_TrainerSpec_BosId
- }
- func (x *TrainerSpec) GetEosId() int32 {
- if x != nil && x.EosId != nil {
- return *x.EosId
- }
- return Default_TrainerSpec_EosId
- }
- func (x *TrainerSpec) GetPadId() int32 {
- if x != nil && x.PadId != nil {
- return *x.PadId
- }
- return Default_TrainerSpec_PadId
- }
- func (x *TrainerSpec) GetUnkPiece() string {
- if x != nil && x.UnkPiece != nil {
- return *x.UnkPiece
- }
- return Default_TrainerSpec_UnkPiece
- }
- func (x *TrainerSpec) GetBosPiece() string {
- if x != nil && x.BosPiece != nil {
- return *x.BosPiece
- }
- return Default_TrainerSpec_BosPiece
- }
- func (x *TrainerSpec) GetEosPiece() string {
- if x != nil && x.EosPiece != nil {
- return *x.EosPiece
- }
- return Default_TrainerSpec_EosPiece
- }
- func (x *TrainerSpec) GetPadPiece() string {
- if x != nil && x.PadPiece != nil {
- return *x.PadPiece
- }
- return Default_TrainerSpec_PadPiece
- }
- func (x *TrainerSpec) GetUnkSurface() string {
- if x != nil && x.UnkSurface != nil {
- return *x.UnkSurface
- }
- return Default_TrainerSpec_UnkSurface
- }
- func (x *TrainerSpec) GetTrainExtremelyLargeCorpus() bool {
- if x != nil && x.TrainExtremelyLargeCorpus != nil {
- return *x.TrainExtremelyLargeCorpus
- }
- return Default_TrainerSpec_TrainExtremelyLargeCorpus
- }
- func (x *TrainerSpec) GetSeedSentencepiecesFile() string {
- if x != nil && x.SeedSentencepiecesFile != nil {
- return *x.SeedSentencepiecesFile
- }
- return Default_TrainerSpec_SeedSentencepiecesFile
- }
- // NormalizerSpec encodes a various parameters for string normalizaiton
- type NormalizerSpec struct {
- state protoimpl.MessageState
- sizeCache protoimpl.SizeCache
- unknownFields protoimpl.UnknownFields
- extensionFields protoimpl.ExtensionFields
- // name of normalization rule.
- Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
- // Pre-compiled normalization rule created by
- // Builder::GetPrecompiledCharsMap() or Builder::CompileCharsMap() method.
- // Usually this field is set by Builder::GetNormalizerSpec() method.
- PrecompiledCharsmap []byte `protobuf:"bytes,2,opt,name=precompiled_charsmap,json=precompiledCharsmap" json:"precompiled_charsmap,omitempty"`
- // Adds dummy whitespace at the beginning of text in order to
- // treat "world" in "world" and "hello world" in the same way.
- AddDummyPrefix *bool `protobuf:"varint,3,opt,name=add_dummy_prefix,json=addDummyPrefix,def=1" json:"add_dummy_prefix,omitempty"`
- // Removes leading, trailing, and duplicate internal whitespace.
- RemoveExtraWhitespaces *bool `protobuf:"varint,4,opt,name=remove_extra_whitespaces,json=removeExtraWhitespaces,def=1" json:"remove_extra_whitespaces,omitempty"`
- // Replaces whitespace with meta symbol.
- // This field must be true to train sentence piece model.
- EscapeWhitespaces *bool `protobuf:"varint,5,opt,name=escape_whitespaces,json=escapeWhitespaces,def=1" json:"escape_whitespaces,omitempty"`
- // Custom normalization rule file in TSV format.
- // https://github.com/google/sentencepiece/blob/master/doc/normalization.md
- // This field is only used in SentencePieceTrainer::Train() method, which
- // compiles the rule into the binary rule stored in `precompiled_charsmap`.
- NormalizationRuleTsv *string `protobuf:"bytes,6,opt,name=normalization_rule_tsv,json=normalizationRuleTsv" json:"normalization_rule_tsv,omitempty"`
- }
- // Default values for NormalizerSpec fields.
- const (
- Default_NormalizerSpec_AddDummyPrefix = bool(true)
- Default_NormalizerSpec_RemoveExtraWhitespaces = bool(true)
- Default_NormalizerSpec_EscapeWhitespaces = bool(true)
- )
- func (x *NormalizerSpec) Reset() {
- *x = NormalizerSpec{}
- if protoimpl.UnsafeEnabled {
- mi := &file_sentencepiece_model_proto_msgTypes[1]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
- }
- }
- func (x *NormalizerSpec) String() string {
- return protoimpl.X.MessageStringOf(x)
- }
- func (*NormalizerSpec) ProtoMessage() {}
- func (x *NormalizerSpec) ProtoReflect() protoreflect.Message {
- mi := &file_sentencepiece_model_proto_msgTypes[1]
- if protoimpl.UnsafeEnabled && x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
- }
- // Deprecated: Use NormalizerSpec.ProtoReflect.Descriptor instead.
- func (*NormalizerSpec) Descriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{1}
- }
- func (x *NormalizerSpec) GetName() string {
- if x != nil && x.Name != nil {
- return *x.Name
- }
- return ""
- }
- func (x *NormalizerSpec) GetPrecompiledCharsmap() []byte {
- if x != nil {
- return x.PrecompiledCharsmap
- }
- return nil
- }
- func (x *NormalizerSpec) GetAddDummyPrefix() bool {
- if x != nil && x.AddDummyPrefix != nil {
- return *x.AddDummyPrefix
- }
- return Default_NormalizerSpec_AddDummyPrefix
- }
- func (x *NormalizerSpec) GetRemoveExtraWhitespaces() bool {
- if x != nil && x.RemoveExtraWhitespaces != nil {
- return *x.RemoveExtraWhitespaces
- }
- return Default_NormalizerSpec_RemoveExtraWhitespaces
- }
- func (x *NormalizerSpec) GetEscapeWhitespaces() bool {
- if x != nil && x.EscapeWhitespaces != nil {
- return *x.EscapeWhitespaces
- }
- return Default_NormalizerSpec_EscapeWhitespaces
- }
- func (x *NormalizerSpec) GetNormalizationRuleTsv() string {
- if x != nil && x.NormalizationRuleTsv != nil {
- return *x.NormalizationRuleTsv
- }
- return ""
- }
- // Proto to store samples for self-testing.
- type SelfTestData struct {
- state protoimpl.MessageState
- sizeCache protoimpl.SizeCache
- unknownFields protoimpl.UnknownFields
- extensionFields protoimpl.ExtensionFields
- Samples []*SelfTestData_Sample `protobuf:"bytes,1,rep,name=samples" json:"samples,omitempty"`
- }
- func (x *SelfTestData) Reset() {
- *x = SelfTestData{}
- if protoimpl.UnsafeEnabled {
- mi := &file_sentencepiece_model_proto_msgTypes[2]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
- }
- }
- func (x *SelfTestData) String() string {
- return protoimpl.X.MessageStringOf(x)
- }
- func (*SelfTestData) ProtoMessage() {}
- func (x *SelfTestData) ProtoReflect() protoreflect.Message {
- mi := &file_sentencepiece_model_proto_msgTypes[2]
- if protoimpl.UnsafeEnabled && x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
- }
- // Deprecated: Use SelfTestData.ProtoReflect.Descriptor instead.
- func (*SelfTestData) Descriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{2}
- }
- func (x *SelfTestData) GetSamples() []*SelfTestData_Sample {
- if x != nil {
- return x.Samples
- }
- return nil
- }
- // ModelProto stores model parameters.
- // SentencePieceProcessor is supposed to be self-contained.
- // All settings/parameters which may change the behavior must be encoded
- // in ModelProto.
- type ModelProto struct {
- state protoimpl.MessageState
- sizeCache protoimpl.SizeCache
- unknownFields protoimpl.UnknownFields
- extensionFields protoimpl.ExtensionFields
- // Sentence pieces with scores.
- Pieces []*ModelProto_SentencePiece `protobuf:"bytes,1,rep,name=pieces" json:"pieces,omitempty"`
- // Spec used to generate this model file.
- TrainerSpec *TrainerSpec `protobuf:"bytes,2,opt,name=trainer_spec,json=trainerSpec" json:"trainer_spec,omitempty"`
- // Spec for text normalization.
- NormalizerSpec *NormalizerSpec `protobuf:"bytes,3,opt,name=normalizer_spec,json=normalizerSpec" json:"normalizer_spec,omitempty"`
- // Stores sample input and its expected segmentation to verify the model.
- SelfTestData *SelfTestData `protobuf:"bytes,4,opt,name=self_test_data,json=selfTestData" json:"self_test_data,omitempty"`
- // Spec for text de-normalization.
- DenormalizerSpec *NormalizerSpec `protobuf:"bytes,5,opt,name=denormalizer_spec,json=denormalizerSpec" json:"denormalizer_spec,omitempty"`
- }
- func (x *ModelProto) Reset() {
- *x = ModelProto{}
- if protoimpl.UnsafeEnabled {
- mi := &file_sentencepiece_model_proto_msgTypes[3]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
- }
- }
- func (x *ModelProto) String() string {
- return protoimpl.X.MessageStringOf(x)
- }
- func (*ModelProto) ProtoMessage() {}
- func (x *ModelProto) ProtoReflect() protoreflect.Message {
- mi := &file_sentencepiece_model_proto_msgTypes[3]
- if protoimpl.UnsafeEnabled && x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
- }
- // Deprecated: Use ModelProto.ProtoReflect.Descriptor instead.
- func (*ModelProto) Descriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{3}
- }
- func (x *ModelProto) GetPieces() []*ModelProto_SentencePiece {
- if x != nil {
- return x.Pieces
- }
- return nil
- }
- func (x *ModelProto) GetTrainerSpec() *TrainerSpec {
- if x != nil {
- return x.TrainerSpec
- }
- return nil
- }
- func (x *ModelProto) GetNormalizerSpec() *NormalizerSpec {
- if x != nil {
- return x.NormalizerSpec
- }
- return nil
- }
- func (x *ModelProto) GetSelfTestData() *SelfTestData {
- if x != nil {
- return x.SelfTestData
- }
- return nil
- }
- func (x *ModelProto) GetDenormalizerSpec() *NormalizerSpec {
- if x != nil {
- return x.DenormalizerSpec
- }
- return nil
- }
- type SelfTestData_Sample struct {
- state protoimpl.MessageState
- sizeCache protoimpl.SizeCache
- unknownFields protoimpl.UnknownFields
- Input *string `protobuf:"bytes,1,opt,name=input" json:"input,omitempty"`
- Expected *string `protobuf:"bytes,2,opt,name=expected" json:"expected,omitempty"`
- }
- func (x *SelfTestData_Sample) Reset() {
- *x = SelfTestData_Sample{}
- if protoimpl.UnsafeEnabled {
- mi := &file_sentencepiece_model_proto_msgTypes[4]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
- }
- }
- func (x *SelfTestData_Sample) String() string {
- return protoimpl.X.MessageStringOf(x)
- }
- func (*SelfTestData_Sample) ProtoMessage() {}
- func (x *SelfTestData_Sample) ProtoReflect() protoreflect.Message {
- mi := &file_sentencepiece_model_proto_msgTypes[4]
- if protoimpl.UnsafeEnabled && x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
- }
- // Deprecated: Use SelfTestData_Sample.ProtoReflect.Descriptor instead.
- func (*SelfTestData_Sample) Descriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{2, 0}
- }
- func (x *SelfTestData_Sample) GetInput() string {
- if x != nil && x.Input != nil {
- return *x.Input
- }
- return ""
- }
- func (x *SelfTestData_Sample) GetExpected() string {
- if x != nil && x.Expected != nil {
- return *x.Expected
- }
- return ""
- }
- type ModelProto_SentencePiece struct {
- state protoimpl.MessageState
- sizeCache protoimpl.SizeCache
- unknownFields protoimpl.UnknownFields
- extensionFields protoimpl.ExtensionFields
- Piece *string `protobuf:"bytes,1,opt,name=piece" json:"piece,omitempty"` // piece must not be empty.
- Score *float32 `protobuf:"fixed32,2,opt,name=score" json:"score,omitempty"`
- Type *ModelProto_SentencePiece_Type `protobuf:"varint,3,opt,name=type,enum=sentencepiece.ModelProto_SentencePiece_Type,def=1" json:"type,omitempty"`
- }
- // Default values for ModelProto_SentencePiece fields.
- const (
- Default_ModelProto_SentencePiece_Type = ModelProto_SentencePiece_NORMAL
- )
- func (x *ModelProto_SentencePiece) Reset() {
- *x = ModelProto_SentencePiece{}
- if protoimpl.UnsafeEnabled {
- mi := &file_sentencepiece_model_proto_msgTypes[5]
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- ms.StoreMessageInfo(mi)
- }
- }
- func (x *ModelProto_SentencePiece) String() string {
- return protoimpl.X.MessageStringOf(x)
- }
- func (*ModelProto_SentencePiece) ProtoMessage() {}
- func (x *ModelProto_SentencePiece) ProtoReflect() protoreflect.Message {
- mi := &file_sentencepiece_model_proto_msgTypes[5]
- if protoimpl.UnsafeEnabled && x != nil {
- ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
- if ms.LoadMessageInfo() == nil {
- ms.StoreMessageInfo(mi)
- }
- return ms
- }
- return mi.MessageOf(x)
- }
- // Deprecated: Use ModelProto_SentencePiece.ProtoReflect.Descriptor instead.
- func (*ModelProto_SentencePiece) Descriptor() ([]byte, []int) {
- return file_sentencepiece_model_proto_rawDescGZIP(), []int{3, 0}
- }
- func (x *ModelProto_SentencePiece) GetPiece() string {
- if x != nil && x.Piece != nil {
- return *x.Piece
- }
- return ""
- }
- func (x *ModelProto_SentencePiece) GetScore() float32 {
- if x != nil && x.Score != nil {
- return *x.Score
- }
- return 0
- }
- func (x *ModelProto_SentencePiece) GetType() ModelProto_SentencePiece_Type {
- if x != nil && x.Type != nil {
- return *x.Type
- }
- return Default_ModelProto_SentencePiece_Type
- }
- var File_sentencepiece_model_proto protoreflect.FileDescriptor
- var file_sentencepiece_model_proto_rawDesc = []byte{
- 0x0a, 0x19, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f,
- 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x73, 0x65, 0x6e,
- 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x22, 0xc6, 0x12, 0x0a, 0x0b, 0x54,
- 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e,
- 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74,
- 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74,
- 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, 0x6f, 0x72,
- 0x6d, 0x61, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x70, 0x72, 0x65,
- 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x6f, 0x64, 0x65, 0x6c,
- 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x4c, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f,
- 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x73, 0x65, 0x6e,
- 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x54, 0x72, 0x61, 0x69, 0x6e,
- 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65,
- 0x3a, 0x07, 0x55, 0x4e, 0x49, 0x47, 0x52, 0x41, 0x4d, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c,
- 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0a, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x5f, 0x73, 0x69,
- 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x04, 0x38, 0x30, 0x30, 0x30, 0x52, 0x09,
- 0x76, 0x6f, 0x63, 0x61, 0x62, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x63, 0x63,
- 0x65, 0x70, 0x74, 0x5f, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x03,
- 0x28, 0x09, 0x52, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x4c, 0x61, 0x6e, 0x67, 0x75, 0x61,
- 0x67, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x73, 0x65, 0x6c, 0x66, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f,
- 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28,
- 0x05, 0x3a, 0x01, 0x30, 0x52, 0x12, 0x73, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x53, 0x61,
- 0x6d, 0x70, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x45, 0x0a, 0x1b, 0x65, 0x6e, 0x61, 0x62,
- 0x6c, 0x65, 0x5f, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f,
- 0x70, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x18, 0x32, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66,
- 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x44, 0x69, 0x66, 0x66,
- 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x12,
- 0x4a, 0x0a, 0x20, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f,
- 0x70, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x5f, 0x6e, 0x6f, 0x69, 0x73, 0x65, 0x5f, 0x6c, 0x65,
- 0x76, 0x65, 0x6c, 0x18, 0x33, 0x20, 0x01, 0x28, 0x02, 0x3a, 0x01, 0x30, 0x52, 0x1d, 0x64, 0x69,
- 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63,
- 0x79, 0x4e, 0x6f, 0x69, 0x73, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x58, 0x0a, 0x27, 0x64,
- 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x70, 0x72, 0x69, 0x76,
- 0x61, 0x63, 0x79, 0x5f, 0x63, 0x6c, 0x69, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x68, 0x72,
- 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x34, 0x20, 0x01, 0x28, 0x04, 0x3a, 0x01, 0x30, 0x52,
- 0x24, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69,
- 0x76, 0x61, 0x63, 0x79, 0x43, 0x6c, 0x69, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x54, 0x68, 0x72, 0x65,
- 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x35, 0x0a, 0x12, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74,
- 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28,
- 0x02, 0x3a, 0x06, 0x30, 0x2e, 0x39, 0x39, 0x39, 0x35, 0x52, 0x11, 0x63, 0x68, 0x61, 0x72, 0x61,
- 0x63, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x12, 0x31, 0x0a, 0x13,
- 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73,
- 0x69, 0x7a, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x3a, 0x01, 0x30, 0x52, 0x11, 0x69, 0x6e,
- 0x70, 0x75, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12,
- 0x3a, 0x0a, 0x16, 0x73, 0x68, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74,
- 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x3a,
- 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x14, 0x73, 0x68, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x49, 0x6e,
- 0x70, 0x75, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x14, 0x6d,
- 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73,
- 0x69, 0x7a, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x42, 0x02, 0x18, 0x01, 0x52, 0x12, 0x6d,
- 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a,
- 0x65, 0x12, 0x38, 0x0a, 0x16, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65,
- 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28,
- 0x05, 0x42, 0x02, 0x18, 0x01, 0x52, 0x14, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53,
- 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x3f, 0x0a, 0x17, 0x73,
- 0x65, 0x65, 0x64, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
- 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x07, 0x31, 0x30,
- 0x30, 0x30, 0x30, 0x30, 0x30, 0x52, 0x15, 0x73, 0x65, 0x65, 0x64, 0x53, 0x65, 0x6e, 0x74, 0x65,
- 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x10,
- 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72,
- 0x18, 0x0f, 0x20, 0x01, 0x28, 0x02, 0x3a, 0x04, 0x30, 0x2e, 0x37, 0x35, 0x52, 0x0f, 0x73, 0x68,
- 0x72, 0x69, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x34, 0x0a,
- 0x13, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6c, 0x65,
- 0x6e, 0x67, 0x74, 0x68, 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x04, 0x34, 0x31, 0x39, 0x32,
- 0x52, 0x11, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x4c, 0x65, 0x6e,
- 0x67, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61,
- 0x64, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x31, 0x36, 0x52, 0x0a, 0x6e, 0x75,
- 0x6d, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x2f, 0x0a, 0x12, 0x6e, 0x75, 0x6d, 0x5f,
- 0x73, 0x75, 0x62, 0x5f, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x11,
- 0x20, 0x01, 0x28, 0x05, 0x3a, 0x01, 0x32, 0x52, 0x10, 0x6e, 0x75, 0x6d, 0x53, 0x75, 0x62, 0x49,
- 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3c, 0x0a, 0x18, 0x6d, 0x61, 0x78,
- 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f, 0x6c,
- 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x31, 0x36, 0x52,
- 0x16, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
- 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x3b, 0x0a, 0x17, 0x73, 0x70, 0x6c, 0x69, 0x74,
- 0x5f, 0x62, 0x79, 0x5f, 0x75, 0x6e, 0x69, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x73, 0x63, 0x72, 0x69,
- 0x70, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x14,
- 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x55, 0x6e, 0x69, 0x63, 0x6f, 0x64, 0x65, 0x53, 0x63,
- 0x72, 0x69, 0x70, 0x74, 0x12, 0x2c, 0x0a, 0x0f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x62, 0x79,
- 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74,
- 0x72, 0x75, 0x65, 0x52, 0x0d, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x4e, 0x75, 0x6d, 0x62,
- 0x65, 0x72, 0x12, 0x34, 0x0a, 0x13, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x77,
- 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x3a,
- 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x11, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x57, 0x68,
- 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x42, 0x0a, 0x1a, 0x74, 0x72, 0x65, 0x61,
- 0x74, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x61, 0x73, 0x5f,
- 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61,
- 0x6c, 0x73, 0x65, 0x52, 0x17, 0x74, 0x72, 0x65, 0x61, 0x74, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73,
- 0x70, 0x61, 0x63, 0x65, 0x41, 0x73, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x46, 0x0a, 0x1c,
- 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65,
- 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x18, 0x1a, 0x20, 0x01,
- 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77,
- 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x69,
- 0x65, 0x63, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x0c, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x64, 0x69,
- 0x67, 0x69, 0x74, 0x73, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73,
- 0x65, 0x52, 0x0b, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x44, 0x69, 0x67, 0x69, 0x74, 0x73, 0x12, 0x3d,
- 0x0a, 0x19, 0x70, 0x72, 0x65, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f,
- 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x35, 0x20, 0x01, 0x28,
- 0x09, 0x3a, 0x00, 0x52, 0x18, 0x70, 0x72, 0x65, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x69, 0x7a, 0x61,
- 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x27, 0x0a,
- 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73,
- 0x18, 0x1e, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53,
- 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x64,
- 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x18, 0x1f,
- 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x75, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65,
- 0x64, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x71, 0x75,
- 0x69, 0x72, 0x65, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x73, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09,
- 0x52, 0x0d, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x43, 0x68, 0x61, 0x72, 0x73, 0x12,
- 0x2a, 0x0a, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b,
- 0x18, 0x23, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0c, 0x62,
- 0x79, 0x74, 0x65, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x47, 0x0a, 0x1d, 0x76,
- 0x6f, 0x63, 0x61, 0x62, 0x75, 0x6c, 0x61, 0x72, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74,
- 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x20, 0x20, 0x01,
- 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x1a, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x75,
- 0x6c, 0x61, 0x72, 0x79, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x50, 0x69, 0x65, 0x63, 0x65, 0x53,
- 0x63, 0x6f, 0x72, 0x65, 0x12, 0x2e, 0x0a, 0x10, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x76, 0x6f, 0x63,
- 0x61, 0x62, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x21, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04,
- 0x74, 0x72, 0x75, 0x65, 0x52, 0x0e, 0x68, 0x61, 0x72, 0x64, 0x56, 0x6f, 0x63, 0x61, 0x62, 0x4c,
- 0x69, 0x6d, 0x69, 0x74, 0x12, 0x29, 0x0a, 0x0d, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x6c, 0x5f,
- 0x76, 0x6f, 0x63, 0x61, 0x62, 0x18, 0x22, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c,
- 0x73, 0x65, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x41, 0x6c, 0x6c, 0x56, 0x6f, 0x63, 0x61, 0x62, 0x12,
- 0x18, 0x0a, 0x06, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x28, 0x20, 0x01, 0x28, 0x05, 0x3a,
- 0x01, 0x30, 0x52, 0x05, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x06, 0x62, 0x6f, 0x73,
- 0x5f, 0x69, 0x64, 0x18, 0x29, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x01, 0x31, 0x52, 0x05, 0x62, 0x6f,
- 0x73, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x06, 0x65, 0x6f, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x2a, 0x20,
- 0x01, 0x28, 0x05, 0x3a, 0x01, 0x32, 0x52, 0x05, 0x65, 0x6f, 0x73, 0x49, 0x64, 0x12, 0x19, 0x0a,
- 0x06, 0x70, 0x61, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x2d,
- 0x31, 0x52, 0x05, 0x70, 0x61, 0x64, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x09, 0x75, 0x6e, 0x6b, 0x5f,
- 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x3c, 0x75, 0x6e,
- 0x6b, 0x3e, 0x52, 0x08, 0x75, 0x6e, 0x6b, 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x09,
- 0x62, 0x6f, 0x73, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2e, 0x20, 0x01, 0x28, 0x09, 0x3a,
- 0x03, 0x3c, 0x73, 0x3e, 0x52, 0x08, 0x62, 0x6f, 0x73, 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x21,
- 0x0a, 0x09, 0x65, 0x6f, 0x73, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2f, 0x20, 0x01, 0x28,
- 0x09, 0x3a, 0x04, 0x3c, 0x2f, 0x73, 0x3e, 0x52, 0x08, 0x65, 0x6f, 0x73, 0x50, 0x69, 0x65, 0x63,
- 0x65, 0x12, 0x22, 0x0a, 0x09, 0x70, 0x61, 0x64, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x30,
- 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x3c, 0x70, 0x61, 0x64, 0x3e, 0x52, 0x08, 0x70, 0x61, 0x64,
- 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x26, 0x0a, 0x0b, 0x75, 0x6e, 0x6b, 0x5f, 0x73, 0x75, 0x72,
- 0x66, 0x61, 0x63, 0x65, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x20, 0xe2, 0x81, 0x87,
- 0x20, 0x52, 0x0a, 0x75, 0x6e, 0x6b, 0x53, 0x75, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x46, 0x0a,
- 0x1c, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79,
- 0x5f, 0x6c, 0x61, 0x72, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x72, 0x70, 0x75, 0x73, 0x18, 0x31, 0x20,
- 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x74, 0x72, 0x61, 0x69,
- 0x6e, 0x45, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79, 0x4c, 0x61, 0x72, 0x67, 0x65, 0x43,
- 0x6f, 0x72, 0x70, 0x75, 0x73, 0x12, 0x3a, 0x0a, 0x18, 0x73, 0x65, 0x65, 0x64, 0x5f, 0x73, 0x65,
- 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x5f, 0x66, 0x69, 0x6c,
- 0x65, 0x18, 0x36, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x00, 0x52, 0x16, 0x73, 0x65, 0x65, 0x64, 0x53,
- 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x46, 0x69, 0x6c,
- 0x65, 0x22, 0x35, 0x0a, 0x09, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b,
- 0x0a, 0x07, 0x55, 0x4e, 0x49, 0x47, 0x52, 0x41, 0x4d, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x42,
- 0x50, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x4f, 0x52, 0x44, 0x10, 0x03, 0x12, 0x08,
- 0x0a, 0x04, 0x43, 0x48, 0x41, 0x52, 0x10, 0x04, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
- 0x80, 0x80, 0x02, 0x22, 0xbd, 0x02, 0x0a, 0x0e, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a,
- 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x31, 0x0a, 0x14, 0x70, 0x72,
- 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x73, 0x6d,
- 0x61, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x70, 0x72, 0x65, 0x63, 0x6f, 0x6d,
- 0x70, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x68, 0x61, 0x72, 0x73, 0x6d, 0x61, 0x70, 0x12, 0x2e, 0x0a,
- 0x10, 0x61, 0x64, 0x64, 0x5f, 0x64, 0x75, 0x6d, 0x6d, 0x79, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69,
- 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x0e, 0x61,
- 0x64, 0x64, 0x44, 0x75, 0x6d, 0x6d, 0x79, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x3e, 0x0a,
- 0x18, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x77, 0x68,
- 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a,
- 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x78, 0x74,
- 0x72, 0x61, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x33, 0x0a,
- 0x12, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61,
- 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52,
- 0x11, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63,
- 0x65, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74,
- 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x5f, 0x74, 0x73, 0x76, 0x18, 0x06, 0x20, 0x01,
- 0x28, 0x09, 0x52, 0x14, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f,
- 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x54, 0x73, 0x76, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
- 0x80, 0x80, 0x02, 0x22, 0x93, 0x01, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74,
- 0x44, 0x61, 0x74, 0x61, 0x12, 0x3c, 0x0a, 0x07, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x18,
- 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65,
- 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x44, 0x61,
- 0x74, 0x61, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x07, 0x73, 0x61, 0x6d, 0x70, 0x6c,
- 0x65, 0x73, 0x1a, 0x3a, 0x0a, 0x06, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05,
- 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70,
- 0x75, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x2a, 0x09,
- 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0xd7, 0x04, 0x0a, 0x0a, 0x4d, 0x6f,
- 0x64, 0x65, 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x3f, 0x0a, 0x06, 0x70, 0x69, 0x65, 0x63,
- 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65,
- 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x72,
- 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x69, 0x65, 0x63,
- 0x65, 0x52, 0x06, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x0c, 0x74, 0x72, 0x61,
- 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
- 0x1a, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e,
- 0x54, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0b, 0x74, 0x72, 0x61,
- 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x46, 0x0a, 0x0f, 0x6e, 0x6f, 0x72, 0x6d,
- 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28,
- 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
- 0x65, 0x2e, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63,
- 0x52, 0x0e, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63,
- 0x12, 0x41, 0x0a, 0x0e, 0x73, 0x65, 0x6c, 0x66, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x64, 0x61,
- 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65,
- 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73,
- 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0c, 0x73, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x44,
- 0x61, 0x74, 0x61, 0x12, 0x4a, 0x0a, 0x11, 0x64, 0x65, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69,
- 0x7a, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d,
- 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4e,
- 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x10, 0x64,
- 0x65, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x1a,
- 0xe6, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x69, 0x65, 0x63,
- 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
- 0x52, 0x05, 0x70, 0x69, 0x65, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65,
- 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x48, 0x0a,
- 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x73, 0x65,
- 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65,
- 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50,
- 0x69, 0x65, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41,
- 0x4c, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x54, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12,
- 0x0a, 0x0a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55,
- 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x54,
- 0x52, 0x4f, 0x4c, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x44, 0x45,
- 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x59, 0x54, 0x45, 0x10,
- 0x06, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x4e, 0x55, 0x53, 0x45, 0x44, 0x10, 0x05, 0x2a, 0x09, 0x08,
- 0xc8, 0x01, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
- 0x80, 0x80, 0x02, 0x42, 0x13, 0x48, 0x03, 0x5a, 0x0f, 0x2e, 0x2f, 0x73, 0x65, 0x6e, 0x74, 0x65,
- 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65,
- }
- var (
- file_sentencepiece_model_proto_rawDescOnce sync.Once
- file_sentencepiece_model_proto_rawDescData = file_sentencepiece_model_proto_rawDesc
- )
- func file_sentencepiece_model_proto_rawDescGZIP() []byte {
- file_sentencepiece_model_proto_rawDescOnce.Do(func() {
- file_sentencepiece_model_proto_rawDescData = protoimpl.X.CompressGZIP(file_sentencepiece_model_proto_rawDescData)
- })
- return file_sentencepiece_model_proto_rawDescData
- }
- var file_sentencepiece_model_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
- var file_sentencepiece_model_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
- var file_sentencepiece_model_proto_goTypes = []interface{}{
- (TrainerSpec_ModelType)(0), // 0: sentencepiece.TrainerSpec.ModelType
- (ModelProto_SentencePiece_Type)(0), // 1: sentencepiece.ModelProto.SentencePiece.Type
- (*TrainerSpec)(nil), // 2: sentencepiece.TrainerSpec
- (*NormalizerSpec)(nil), // 3: sentencepiece.NormalizerSpec
- (*SelfTestData)(nil), // 4: sentencepiece.SelfTestData
- (*ModelProto)(nil), // 5: sentencepiece.ModelProto
- (*SelfTestData_Sample)(nil), // 6: sentencepiece.SelfTestData.Sample
- (*ModelProto_SentencePiece)(nil), // 7: sentencepiece.ModelProto.SentencePiece
- }
- var file_sentencepiece_model_proto_depIdxs = []int32{
- 0, // 0: sentencepiece.TrainerSpec.model_type:type_name -> sentencepiece.TrainerSpec.ModelType
- 6, // 1: sentencepiece.SelfTestData.samples:type_name -> sentencepiece.SelfTestData.Sample
- 7, // 2: sentencepiece.ModelProto.pieces:type_name -> sentencepiece.ModelProto.SentencePiece
- 2, // 3: sentencepiece.ModelProto.trainer_spec:type_name -> sentencepiece.TrainerSpec
- 3, // 4: sentencepiece.ModelProto.normalizer_spec:type_name -> sentencepiece.NormalizerSpec
- 4, // 5: sentencepiece.ModelProto.self_test_data:type_name -> sentencepiece.SelfTestData
- 3, // 6: sentencepiece.ModelProto.denormalizer_spec:type_name -> sentencepiece.NormalizerSpec
- 1, // 7: sentencepiece.ModelProto.SentencePiece.type:type_name -> sentencepiece.ModelProto.SentencePiece.Type
- 8, // [8:8] is the sub-list for method output_type
- 8, // [8:8] is the sub-list for method input_type
- 8, // [8:8] is the sub-list for extension type_name
- 8, // [8:8] is the sub-list for extension extendee
- 0, // [0:8] is the sub-list for field type_name
- }
- func init() { file_sentencepiece_model_proto_init() }
- func file_sentencepiece_model_proto_init() {
- if File_sentencepiece_model_proto != nil {
- return
- }
- if !protoimpl.UnsafeEnabled {
- file_sentencepiece_model_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*TrainerSpec); i {
- case 0:
- return &v.state
- case 1:
- return &v.sizeCache
- case 2:
- return &v.unknownFields
- case 3:
- return &v.extensionFields
- default:
- return nil
- }
- }
- file_sentencepiece_model_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*NormalizerSpec); i {
- case 0:
- return &v.state
- case 1:
- return &v.sizeCache
- case 2:
- return &v.unknownFields
- case 3:
- return &v.extensionFields
- default:
- return nil
- }
- }
- file_sentencepiece_model_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*SelfTestData); i {
- case 0:
- return &v.state
- case 1:
- return &v.sizeCache
- case 2:
- return &v.unknownFields
- case 3:
- return &v.extensionFields
- default:
- return nil
- }
- }
- file_sentencepiece_model_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*ModelProto); i {
- case 0:
- return &v.state
- case 1:
- return &v.sizeCache
- case 2:
- return &v.unknownFields
- case 3:
- return &v.extensionFields
- default:
- return nil
- }
- }
- file_sentencepiece_model_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*SelfTestData_Sample); i {
- case 0:
- return &v.state
- case 1:
- return &v.sizeCache
- case 2:
- return &v.unknownFields
- default:
- return nil
- }
- }
- file_sentencepiece_model_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*ModelProto_SentencePiece); i {
- case 0:
- return &v.state
- case 1:
- return &v.sizeCache
- case 2:
- return &v.unknownFields
- case 3:
- return &v.extensionFields
- default:
- return nil
- }
- }
- }
- type x struct{}
- out := protoimpl.TypeBuilder{
- File: protoimpl.DescBuilder{
- GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
- RawDescriptor: file_sentencepiece_model_proto_rawDesc,
- NumEnums: 2,
- NumMessages: 6,
- NumExtensions: 0,
- NumServices: 0,
- },
- GoTypes: file_sentencepiece_model_proto_goTypes,
- DependencyIndexes: file_sentencepiece_model_proto_depIdxs,
- EnumInfos: file_sentencepiece_model_proto_enumTypes,
- MessageInfos: file_sentencepiece_model_proto_msgTypes,
- }.Build()
- File_sentencepiece_model_proto = out.File
- file_sentencepiece_model_proto_rawDesc = nil
- file_sentencepiece_model_proto_goTypes = nil
- file_sentencepiece_model_proto_depIdxs = nil
- }
|