sentencepiece_model.pb.go 66 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497
  1. // Copyright 2016 Google Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.!
  14. // Code generated by protoc-gen-go. DO NOT EDIT.
  15. // versions:
  16. // protoc-gen-go v1.32.0
  17. // protoc v4.25.2
  18. // source: sentencepiece_model.proto
  19. package sentencepiece
  20. import (
  21. protoreflect "google.golang.org/protobuf/reflect/protoreflect"
  22. protoimpl "google.golang.org/protobuf/runtime/protoimpl"
  23. reflect "reflect"
  24. sync "sync"
  25. )
  26. const (
  27. // Verify that this generated code is sufficiently up-to-date.
  28. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
  29. // Verify that runtime/protoimpl is sufficiently up-to-date.
  30. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
  31. )
  32. // Model type. only have UNIGRAM now.
  33. type TrainerSpec_ModelType int32
  34. const (
  35. TrainerSpec_UNIGRAM TrainerSpec_ModelType = 1 // Unigram language model with dynamic algorithm
  36. TrainerSpec_BPE TrainerSpec_ModelType = 2 // Byte Pair Encoding
  37. TrainerSpec_WORD TrainerSpec_ModelType = 3 // Delimitered by whitespace.
  38. TrainerSpec_CHAR TrainerSpec_ModelType = 4 // tokenizes into character sequence
  39. )
  40. // Enum value maps for TrainerSpec_ModelType.
  41. var (
  42. TrainerSpec_ModelType_name = map[int32]string{
  43. 1: "UNIGRAM",
  44. 2: "BPE",
  45. 3: "WORD",
  46. 4: "CHAR",
  47. }
  48. TrainerSpec_ModelType_value = map[string]int32{
  49. "UNIGRAM": 1,
  50. "BPE": 2,
  51. "WORD": 3,
  52. "CHAR": 4,
  53. }
  54. )
  55. func (x TrainerSpec_ModelType) Enum() *TrainerSpec_ModelType {
  56. p := new(TrainerSpec_ModelType)
  57. *p = x
  58. return p
  59. }
  60. func (x TrainerSpec_ModelType) String() string {
  61. return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
  62. }
  63. func (TrainerSpec_ModelType) Descriptor() protoreflect.EnumDescriptor {
  64. return file_sentencepiece_model_proto_enumTypes[0].Descriptor()
  65. }
  66. func (TrainerSpec_ModelType) Type() protoreflect.EnumType {
  67. return &file_sentencepiece_model_proto_enumTypes[0]
  68. }
  69. func (x TrainerSpec_ModelType) Number() protoreflect.EnumNumber {
  70. return protoreflect.EnumNumber(x)
  71. }
  72. // Deprecated: Do not use.
  73. func (x *TrainerSpec_ModelType) UnmarshalJSON(b []byte) error {
  74. num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b)
  75. if err != nil {
  76. return err
  77. }
  78. *x = TrainerSpec_ModelType(num)
  79. return nil
  80. }
  81. // Deprecated: Use TrainerSpec_ModelType.Descriptor instead.
  82. func (TrainerSpec_ModelType) EnumDescriptor() ([]byte, []int) {
  83. return file_sentencepiece_model_proto_rawDescGZIP(), []int{0, 0}
  84. }
  85. type ModelProto_SentencePiece_Type int32
  86. const (
  87. ModelProto_SentencePiece_NORMAL ModelProto_SentencePiece_Type = 1 // normal symbol
  88. ModelProto_SentencePiece_UNKNOWN ModelProto_SentencePiece_Type = 2 // unknown symbol. only <unk> for now.
  89. ModelProto_SentencePiece_CONTROL ModelProto_SentencePiece_Type = 3 // control symbols. </s>, <s>, <2ja> etc.
  90. ModelProto_SentencePiece_USER_DEFINED ModelProto_SentencePiece_Type = 4 // user defined symbols.
  91. // Typical usage of USER_DEFINED symbol
  92. // is placeholder.
  93. ModelProto_SentencePiece_BYTE ModelProto_SentencePiece_Type = 6 // byte symbols. Used when `byte_fallback` is true.
  94. ModelProto_SentencePiece_UNUSED ModelProto_SentencePiece_Type = 5 // this piece is not used.
  95. )
  96. // Enum value maps for ModelProto_SentencePiece_Type.
  97. var (
  98. ModelProto_SentencePiece_Type_name = map[int32]string{
  99. 1: "NORMAL",
  100. 2: "UNKNOWN",
  101. 3: "CONTROL",
  102. 4: "USER_DEFINED",
  103. 6: "BYTE",
  104. 5: "UNUSED",
  105. }
  106. ModelProto_SentencePiece_Type_value = map[string]int32{
  107. "NORMAL": 1,
  108. "UNKNOWN": 2,
  109. "CONTROL": 3,
  110. "USER_DEFINED": 4,
  111. "BYTE": 6,
  112. "UNUSED": 5,
  113. }
  114. )
  115. func (x ModelProto_SentencePiece_Type) Enum() *ModelProto_SentencePiece_Type {
  116. p := new(ModelProto_SentencePiece_Type)
  117. *p = x
  118. return p
  119. }
  120. func (x ModelProto_SentencePiece_Type) String() string {
  121. return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
  122. }
  123. func (ModelProto_SentencePiece_Type) Descriptor() protoreflect.EnumDescriptor {
  124. return file_sentencepiece_model_proto_enumTypes[1].Descriptor()
  125. }
  126. func (ModelProto_SentencePiece_Type) Type() protoreflect.EnumType {
  127. return &file_sentencepiece_model_proto_enumTypes[1]
  128. }
  129. func (x ModelProto_SentencePiece_Type) Number() protoreflect.EnumNumber {
  130. return protoreflect.EnumNumber(x)
  131. }
  132. // Deprecated: Do not use.
  133. func (x *ModelProto_SentencePiece_Type) UnmarshalJSON(b []byte) error {
  134. num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b)
  135. if err != nil {
  136. return err
  137. }
  138. *x = ModelProto_SentencePiece_Type(num)
  139. return nil
  140. }
  141. // Deprecated: Use ModelProto_SentencePiece_Type.Descriptor instead.
  142. func (ModelProto_SentencePiece_Type) EnumDescriptor() ([]byte, []int) {
  143. return file_sentencepiece_model_proto_rawDescGZIP(), []int{3, 0, 0}
  144. }
  145. // TrainerSpec encodes a various parameters for SentencePiece training.
  146. // Next id: 55
  147. type TrainerSpec struct {
  148. state protoimpl.MessageState
  149. sizeCache protoimpl.SizeCache
  150. unknownFields protoimpl.UnknownFields
  151. extensionFields protoimpl.ExtensionFields
  152. // /////////////////////////////////////////////////////////////////
  153. // General parameters
  154. //
  155. // Input corpus files.
  156. //
  157. // Trainer accepts the following two formats:
  158. // A) Monolingual: plain text, one sentence per line.
  159. // B) Bilingual: TSV, source sentence <tab> target sentence
  160. // When bilingual data is passed, shared vocabulary model is built.
  161. // Note that the input file must be raw corpus, not a preprocessed corpus.
  162. // Trainer only loads the first `input_sentence_size` sentences specified
  163. // with this parameter.
  164. Input []string `protobuf:"bytes,1,rep,name=input" json:"input,omitempty"`
  165. // Input corpus format:
  166. // "text": one-sentence-per-line text format (default)
  167. // "tsv": sentence <tab> freq
  168. InputFormat *string `protobuf:"bytes,7,opt,name=input_format,json=inputFormat" json:"input_format,omitempty"`
  169. // Output model file prefix.
  170. // <model_prefix>.model and <model_prefix>.vocab are generated.
  171. ModelPrefix *string `protobuf:"bytes,2,opt,name=model_prefix,json=modelPrefix" json:"model_prefix,omitempty"`
  172. ModelType *TrainerSpec_ModelType `protobuf:"varint,3,opt,name=model_type,json=modelType,enum=sentencepiece.TrainerSpec_ModelType,def=1" json:"model_type,omitempty"`
  173. // Vocabulary size. 8k is the default size.
  174. VocabSize *int32 `protobuf:"varint,4,opt,name=vocab_size,json=vocabSize,def=8000" json:"vocab_size,omitempty"`
  175. // List of the languages this model can accept.
  176. // Since the model is language-agnostic, this field is used as a reference.
  177. AcceptLanguage []string `protobuf:"bytes,5,rep,name=accept_language,json=acceptLanguage" json:"accept_language,omitempty"`
  178. // Size of self-test samples, which are encoded in the model file.
  179. SelfTestSampleSize *int32 `protobuf:"varint,6,opt,name=self_test_sample_size,json=selfTestSampleSize,def=0" json:"self_test_sample_size,omitempty"`
  180. // Whether to use DP version of sentencepiece. Use it with TSV input format
  181. // (requires precomputed word tab counts to work).
  182. EnableDifferentialPrivacy *bool `protobuf:"varint,50,opt,name=enable_differential_privacy,json=enableDifferentialPrivacy,def=0" json:"enable_differential_privacy,omitempty"`
  183. // Set these parameters if you need DP version of sentencepiece.
  184. // std of noise to add.
  185. DifferentialPrivacyNoiseLevel *float32 `protobuf:"fixed32,51,opt,name=differential_privacy_noise_level,json=differentialPrivacyNoiseLevel,def=0" json:"differential_privacy_noise_level,omitempty"`
  186. // Clipping threshold to apply after adding noise. All the words with
  187. // frequency less than this value are dropped.
  188. DifferentialPrivacyClippingThreshold *uint64 `protobuf:"varint,52,opt,name=differential_privacy_clipping_threshold,json=differentialPrivacyClippingThreshold,def=0" json:"differential_privacy_clipping_threshold,omitempty"`
  189. // /////////////////////////////////////////////////////////////////
  190. // Training parameters.
  191. //
  192. // Uses characters which cover the corpus with the ratio of `chars_coverage`.
  193. // This parameter determines the set of basic Alphabet of sentence piece.
  194. // 1.0 - `chars_coverage` characters are treated as UNK.
  195. // See also required_chars field.
  196. CharacterCoverage *float32 `protobuf:"fixed32,10,opt,name=character_coverage,json=characterCoverage,def=0.9995" json:"character_coverage,omitempty"`
  197. // Maximum size of sentences the trainer loads from `input` parameter.
  198. // Trainer simply loads the `input` files in sequence.
  199. // It is better to shuffle the input corpus randomly.
  200. InputSentenceSize *uint64 `protobuf:"varint,11,opt,name=input_sentence_size,json=inputSentenceSize,def=0" json:"input_sentence_size,omitempty"`
  201. ShuffleInputSentence *bool `protobuf:"varint,19,opt,name=shuffle_input_sentence,json=shuffleInputSentence,def=1" json:"shuffle_input_sentence,omitempty"`
  202. // Maximum size of sentences to make seed sentence pieces.
  203. // Extended suffix array is constructed to extract frequent
  204. // sub-strings from the corpus. This uses 20N working space,
  205. // where N is the size of corpus.
  206. //
  207. // Deprecated: Marked as deprecated in sentencepiece_model.proto.
  208. MiningSentenceSize *int32 `protobuf:"varint,12,opt,name=mining_sentence_size,json=miningSentenceSize" json:"mining_sentence_size,omitempty"`
  209. // Maximum size of sentences to train sentence pieces.
  210. //
  211. // Deprecated: Marked as deprecated in sentencepiece_model.proto.
  212. TrainingSentenceSize *int32 `protobuf:"varint,13,opt,name=training_sentence_size,json=trainingSentenceSize" json:"training_sentence_size,omitempty"`
  213. // The size of seed sentencepieces.
  214. // `seed_sentencepiece_size` must be larger than `vocab_size`.
  215. SeedSentencepieceSize *int32 `protobuf:"varint,14,opt,name=seed_sentencepiece_size,json=seedSentencepieceSize,def=1000000" json:"seed_sentencepiece_size,omitempty"`
  216. // In every EM sub-iterations, keeps top
  217. // `shrinking_factor` * `current sentencepieces size` with respect to
  218. // the loss of the sentence piece. This value should be smaller than 1.0.
  219. ShrinkingFactor *float32 `protobuf:"fixed32,15,opt,name=shrinking_factor,json=shrinkingFactor,def=0.75" json:"shrinking_factor,omitempty"`
  220. // The maximum sentence length in byte. The sentences with the length
  221. // larger than `max_sentence_length` is simply ignored.
  222. // Longer input tends to bring the following risks:
  223. // - Overflow during EM training (unigram language model only)
  224. // - Performance drop because of O(n log n) cost in BPE.
  225. MaxSentenceLength *int32 `protobuf:"varint,18,opt,name=max_sentence_length,json=maxSentenceLength,def=4192" json:"max_sentence_length,omitempty"`
  226. // Number of threads in the training.
  227. NumThreads *int32 `protobuf:"varint,16,opt,name=num_threads,json=numThreads,def=16" json:"num_threads,omitempty"`
  228. // Number of EM sub iterations.
  229. NumSubIterations *int32 `protobuf:"varint,17,opt,name=num_sub_iterations,json=numSubIterations,def=2" json:"num_sub_iterations,omitempty"`
  230. // /////////////////////////////////////////////////////////////////
  231. // SentencePiece parameters which control the shapes of sentence piece.
  232. //
  233. // Maximum length of sentencepiece.
  234. MaxSentencepieceLength *int32 `protobuf:"varint,20,opt,name=max_sentencepiece_length,json=maxSentencepieceLength,def=16" json:"max_sentencepiece_length,omitempty"`
  235. // Uses Unicode script to split sentence pieces.
  236. // When `split_by_unicode_script` is true, we do not allow sentence piece to
  237. // include multiple Unicode scripts, e.g. "F1" is not a valid piece.
  238. // Exception: CJ characters (Hiragana/Katakana/Han) are all handled
  239. // as one script type, since Japanese word can consist of multiple scripts.
  240. // This exception is always applied regardless of the accept-language
  241. // parameter.
  242. SplitByUnicodeScript *bool `protobuf:"varint,21,opt,name=split_by_unicode_script,json=splitByUnicodeScript,def=1" json:"split_by_unicode_script,omitempty"`
  243. // When `split_by_number` is true, put a boundary between number and
  244. // non-number transition. If we want to treat "F1" is one token, set this flag
  245. // to be false.
  246. SplitByNumber *bool `protobuf:"varint,23,opt,name=split_by_number,json=splitByNumber,def=1" json:"split_by_number,omitempty"`
  247. // Use a white space to split sentence pieces.
  248. // When `split_by_whitespace` is false, we may have the piece containing
  249. // a white space in the middle. e.g., "in_the".
  250. SplitByWhitespace *bool `protobuf:"varint,22,opt,name=split_by_whitespace,json=splitByWhitespace,def=1" json:"split_by_whitespace,omitempty"`
  251. // Adds whitespace symbol (_) as a suffix instead of prefix. e.g., _hello =>
  252. // hello_. When `treat_whitespace_as_suffix` is true,
  253. // NormalizerSpec::add_dummy_prefix will add the dummy whitespace to the end
  254. // of sentence.
  255. TreatWhitespaceAsSuffix *bool `protobuf:"varint,24,opt,name=treat_whitespace_as_suffix,json=treatWhitespaceAsSuffix,def=0" json:"treat_whitespace_as_suffix,omitempty"`
  256. // Allows pieces that only contain whitespaces instead of appearing only as
  257. // prefix or suffix of other pieces.
  258. AllowWhitespaceOnlyPieces *bool `protobuf:"varint,26,opt,name=allow_whitespace_only_pieces,json=allowWhitespaceOnlyPieces,def=0" json:"allow_whitespace_only_pieces,omitempty"`
  259. // Split all digits (0-9) into separate pieces.
  260. SplitDigits *bool `protobuf:"varint,25,opt,name=split_digits,json=splitDigits,def=0" json:"split_digits,omitempty"`
  261. // Defines the pre-tokenization delimiter.
  262. // When specified, no pieces crossing this delimiter is not included
  263. // in the vocab. Then the delimiter string is virtually ignored
  264. // during the training. This field can allows constraints on the vocabulary
  265. // selection. Note that this field is available on unigram mode.
  266. PretokenizationDelimiter *string `protobuf:"bytes,53,opt,name=pretokenization_delimiter,json=pretokenizationDelimiter,def=" json:"pretokenization_delimiter,omitempty"`
  267. // /////////////////////////////////////////////////////////////////
  268. // Vocabulary management
  269. //
  270. // Defines control symbols used as an indicator to
  271. // change the behavior of the decoder. <s> and </s> are pre-defined.
  272. // We can use this field to encode various meta information,
  273. // including language indicator in multilingual model.
  274. // These symbols are not visible to users, but visible to
  275. // the decoder. Note that when the input sentence contains control symbols,
  276. // they are not treated as one token, but segmented into normal pieces.
  277. // Control symbols must be inserted independently from the segmentation.
  278. ControlSymbols []string `protobuf:"bytes,30,rep,name=control_symbols,json=controlSymbols" json:"control_symbols,omitempty"`
  279. // Defines user defined symbols.
  280. // These symbols are added with extremely high score
  281. // so they are always treated as one unique symbol in any context.
  282. // Typical usage of user_defined_symbols is placeholder for named entities.
  283. UserDefinedSymbols []string `protobuf:"bytes,31,rep,name=user_defined_symbols,json=userDefinedSymbols" json:"user_defined_symbols,omitempty"`
  284. // Defines required characters. Each UTF8 character in this string is included
  285. // in the character set regardless of character_coverage value. Unlike
  286. // user_defined_symbols, these characters have scores based on the frequency
  287. // on input sentences, and the model can form subwords using characters
  288. // in this field.
  289. RequiredChars *string `protobuf:"bytes,36,opt,name=required_chars,json=requiredChars" json:"required_chars,omitempty"`
  290. // Decomposes unknown pieces into UTF-8 bytes.
  291. ByteFallback *bool `protobuf:"varint,35,opt,name=byte_fallback,json=byteFallback,def=0" json:"byte_fallback,omitempty"`
  292. // When creating the vocabulary file, defines whether or not to additionally
  293. // output the score for each piece.
  294. VocabularyOutputPieceScore *bool `protobuf:"varint,32,opt,name=vocabulary_output_piece_score,json=vocabularyOutputPieceScore,def=1" json:"vocabulary_output_piece_score,omitempty"`
  295. // `vocab_size` is treated as hard limit. Crash if
  296. // the model can not produce the vocab of size `vocab_size`,
  297. // When `hard_vocab_limit` is false, vocab_size is treated
  298. // as soft limit. Note that when model_type=char,
  299. // always assumes hard_vocab_limit = false.
  300. HardVocabLimit *bool `protobuf:"varint,33,opt,name=hard_vocab_limit,json=hardVocabLimit,def=1" json:"hard_vocab_limit,omitempty"`
  301. // use all symbols for vocab extraction. This flag is valid
  302. // if model type is either CHAR or WORD
  303. UseAllVocab *bool `protobuf:"varint,34,opt,name=use_all_vocab,json=useAllVocab,def=0" json:"use_all_vocab,omitempty"`
  304. // /////////////////////////////////////////////////////////////////
  305. // Reserved special meta tokens.
  306. // * -1 is not used.
  307. // * unk_id must not be -1.
  308. // Id must start with 0 and be contiguous.
  309. UnkId *int32 `protobuf:"varint,40,opt,name=unk_id,json=unkId,def=0" json:"unk_id,omitempty"` // <unk>
  310. BosId *int32 `protobuf:"varint,41,opt,name=bos_id,json=bosId,def=1" json:"bos_id,omitempty"` // <s>
  311. EosId *int32 `protobuf:"varint,42,opt,name=eos_id,json=eosId,def=2" json:"eos_id,omitempty"` // </s>
  312. PadId *int32 `protobuf:"varint,43,opt,name=pad_id,json=padId,def=-1" json:"pad_id,omitempty"` // <pad> (padding)
  313. UnkPiece *string `protobuf:"bytes,45,opt,name=unk_piece,json=unkPiece,def=<unk>" json:"unk_piece,omitempty"`
  314. BosPiece *string `protobuf:"bytes,46,opt,name=bos_piece,json=bosPiece,def=<s>" json:"bos_piece,omitempty"`
  315. EosPiece *string `protobuf:"bytes,47,opt,name=eos_piece,json=eosPiece,def=</s>" json:"eos_piece,omitempty"`
  316. PadPiece *string `protobuf:"bytes,48,opt,name=pad_piece,json=padPiece,def=<pad>" json:"pad_piece,omitempty"`
  317. // Encodes <unk> into U+2047 (DOUBLE QUESTION MARK),
  318. // since this character can be useful both for user and
  319. // developer. We can easily figure out that <unk> is emitted.
  320. UnkSurface *string `protobuf:"bytes,44,opt,name=unk_surface,json=unkSurface,def= ⁇ " json:"unk_surface,omitempty"`
  321. // Increase bit depth to allow unigram model training on large
  322. // (>10M sentences) corpora. A Side-effect of enabling this flag
  323. // is increased memory usage.
  324. TrainExtremelyLargeCorpus *bool `protobuf:"varint,49,opt,name=train_extremely_large_corpus,json=trainExtremelyLargeCorpus,def=0" json:"train_extremely_large_corpus,omitempty"`
  325. // Path to a seed sentencepieces file, with one tab-separated
  326. // seed sentencepiece <tab> frequency per line.
  327. SeedSentencepiecesFile *string `protobuf:"bytes,54,opt,name=seed_sentencepieces_file,json=seedSentencepiecesFile,def=" json:"seed_sentencepieces_file,omitempty"`
  328. }
  329. // Default values for TrainerSpec fields.
  330. const (
  331. Default_TrainerSpec_ModelType = TrainerSpec_UNIGRAM
  332. Default_TrainerSpec_VocabSize = int32(8000)
  333. Default_TrainerSpec_SelfTestSampleSize = int32(0)
  334. Default_TrainerSpec_EnableDifferentialPrivacy = bool(false)
  335. Default_TrainerSpec_DifferentialPrivacyNoiseLevel = float32(0)
  336. Default_TrainerSpec_DifferentialPrivacyClippingThreshold = uint64(0)
  337. Default_TrainerSpec_CharacterCoverage = float32(0.9994999766349792)
  338. Default_TrainerSpec_InputSentenceSize = uint64(0)
  339. Default_TrainerSpec_ShuffleInputSentence = bool(true)
  340. Default_TrainerSpec_SeedSentencepieceSize = int32(1000000)
  341. Default_TrainerSpec_ShrinkingFactor = float32(0.75)
  342. Default_TrainerSpec_MaxSentenceLength = int32(4192)
  343. Default_TrainerSpec_NumThreads = int32(16)
  344. Default_TrainerSpec_NumSubIterations = int32(2)
  345. Default_TrainerSpec_MaxSentencepieceLength = int32(16)
  346. Default_TrainerSpec_SplitByUnicodeScript = bool(true)
  347. Default_TrainerSpec_SplitByNumber = bool(true)
  348. Default_TrainerSpec_SplitByWhitespace = bool(true)
  349. Default_TrainerSpec_TreatWhitespaceAsSuffix = bool(false)
  350. Default_TrainerSpec_AllowWhitespaceOnlyPieces = bool(false)
  351. Default_TrainerSpec_SplitDigits = bool(false)
  352. Default_TrainerSpec_PretokenizationDelimiter = string("")
  353. Default_TrainerSpec_ByteFallback = bool(false)
  354. Default_TrainerSpec_VocabularyOutputPieceScore = bool(true)
  355. Default_TrainerSpec_HardVocabLimit = bool(true)
  356. Default_TrainerSpec_UseAllVocab = bool(false)
  357. Default_TrainerSpec_UnkId = int32(0)
  358. Default_TrainerSpec_BosId = int32(1)
  359. Default_TrainerSpec_EosId = int32(2)
  360. Default_TrainerSpec_PadId = int32(-1)
  361. Default_TrainerSpec_UnkPiece = string("<unk>")
  362. Default_TrainerSpec_BosPiece = string("<s>")
  363. Default_TrainerSpec_EosPiece = string("</s>")
  364. Default_TrainerSpec_PadPiece = string("<pad>")
  365. Default_TrainerSpec_UnkSurface = string(" ⁇ ")
  366. Default_TrainerSpec_TrainExtremelyLargeCorpus = bool(false)
  367. Default_TrainerSpec_SeedSentencepiecesFile = string("")
  368. )
  369. func (x *TrainerSpec) Reset() {
  370. *x = TrainerSpec{}
  371. if protoimpl.UnsafeEnabled {
  372. mi := &file_sentencepiece_model_proto_msgTypes[0]
  373. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  374. ms.StoreMessageInfo(mi)
  375. }
  376. }
  377. func (x *TrainerSpec) String() string {
  378. return protoimpl.X.MessageStringOf(x)
  379. }
  380. func (*TrainerSpec) ProtoMessage() {}
  381. func (x *TrainerSpec) ProtoReflect() protoreflect.Message {
  382. mi := &file_sentencepiece_model_proto_msgTypes[0]
  383. if protoimpl.UnsafeEnabled && x != nil {
  384. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  385. if ms.LoadMessageInfo() == nil {
  386. ms.StoreMessageInfo(mi)
  387. }
  388. return ms
  389. }
  390. return mi.MessageOf(x)
  391. }
  392. // Deprecated: Use TrainerSpec.ProtoReflect.Descriptor instead.
  393. func (*TrainerSpec) Descriptor() ([]byte, []int) {
  394. return file_sentencepiece_model_proto_rawDescGZIP(), []int{0}
  395. }
  396. func (x *TrainerSpec) GetInput() []string {
  397. if x != nil {
  398. return x.Input
  399. }
  400. return nil
  401. }
  402. func (x *TrainerSpec) GetInputFormat() string {
  403. if x != nil && x.InputFormat != nil {
  404. return *x.InputFormat
  405. }
  406. return ""
  407. }
  408. func (x *TrainerSpec) GetModelPrefix() string {
  409. if x != nil && x.ModelPrefix != nil {
  410. return *x.ModelPrefix
  411. }
  412. return ""
  413. }
  414. func (x *TrainerSpec) GetModelType() TrainerSpec_ModelType {
  415. if x != nil && x.ModelType != nil {
  416. return *x.ModelType
  417. }
  418. return Default_TrainerSpec_ModelType
  419. }
  420. func (x *TrainerSpec) GetVocabSize() int32 {
  421. if x != nil && x.VocabSize != nil {
  422. return *x.VocabSize
  423. }
  424. return Default_TrainerSpec_VocabSize
  425. }
  426. func (x *TrainerSpec) GetAcceptLanguage() []string {
  427. if x != nil {
  428. return x.AcceptLanguage
  429. }
  430. return nil
  431. }
  432. func (x *TrainerSpec) GetSelfTestSampleSize() int32 {
  433. if x != nil && x.SelfTestSampleSize != nil {
  434. return *x.SelfTestSampleSize
  435. }
  436. return Default_TrainerSpec_SelfTestSampleSize
  437. }
  438. func (x *TrainerSpec) GetEnableDifferentialPrivacy() bool {
  439. if x != nil && x.EnableDifferentialPrivacy != nil {
  440. return *x.EnableDifferentialPrivacy
  441. }
  442. return Default_TrainerSpec_EnableDifferentialPrivacy
  443. }
  444. func (x *TrainerSpec) GetDifferentialPrivacyNoiseLevel() float32 {
  445. if x != nil && x.DifferentialPrivacyNoiseLevel != nil {
  446. return *x.DifferentialPrivacyNoiseLevel
  447. }
  448. return Default_TrainerSpec_DifferentialPrivacyNoiseLevel
  449. }
  450. func (x *TrainerSpec) GetDifferentialPrivacyClippingThreshold() uint64 {
  451. if x != nil && x.DifferentialPrivacyClippingThreshold != nil {
  452. return *x.DifferentialPrivacyClippingThreshold
  453. }
  454. return Default_TrainerSpec_DifferentialPrivacyClippingThreshold
  455. }
  456. func (x *TrainerSpec) GetCharacterCoverage() float32 {
  457. if x != nil && x.CharacterCoverage != nil {
  458. return *x.CharacterCoverage
  459. }
  460. return Default_TrainerSpec_CharacterCoverage
  461. }
  462. func (x *TrainerSpec) GetInputSentenceSize() uint64 {
  463. if x != nil && x.InputSentenceSize != nil {
  464. return *x.InputSentenceSize
  465. }
  466. return Default_TrainerSpec_InputSentenceSize
  467. }
  468. func (x *TrainerSpec) GetShuffleInputSentence() bool {
  469. if x != nil && x.ShuffleInputSentence != nil {
  470. return *x.ShuffleInputSentence
  471. }
  472. return Default_TrainerSpec_ShuffleInputSentence
  473. }
  474. // Deprecated: Marked as deprecated in sentencepiece_model.proto.
  475. func (x *TrainerSpec) GetMiningSentenceSize() int32 {
  476. if x != nil && x.MiningSentenceSize != nil {
  477. return *x.MiningSentenceSize
  478. }
  479. return 0
  480. }
  481. // Deprecated: Marked as deprecated in sentencepiece_model.proto.
  482. func (x *TrainerSpec) GetTrainingSentenceSize() int32 {
  483. if x != nil && x.TrainingSentenceSize != nil {
  484. return *x.TrainingSentenceSize
  485. }
  486. return 0
  487. }
  488. func (x *TrainerSpec) GetSeedSentencepieceSize() int32 {
  489. if x != nil && x.SeedSentencepieceSize != nil {
  490. return *x.SeedSentencepieceSize
  491. }
  492. return Default_TrainerSpec_SeedSentencepieceSize
  493. }
  494. func (x *TrainerSpec) GetShrinkingFactor() float32 {
  495. if x != nil && x.ShrinkingFactor != nil {
  496. return *x.ShrinkingFactor
  497. }
  498. return Default_TrainerSpec_ShrinkingFactor
  499. }
  500. func (x *TrainerSpec) GetMaxSentenceLength() int32 {
  501. if x != nil && x.MaxSentenceLength != nil {
  502. return *x.MaxSentenceLength
  503. }
  504. return Default_TrainerSpec_MaxSentenceLength
  505. }
  506. func (x *TrainerSpec) GetNumThreads() int32 {
  507. if x != nil && x.NumThreads != nil {
  508. return *x.NumThreads
  509. }
  510. return Default_TrainerSpec_NumThreads
  511. }
  512. func (x *TrainerSpec) GetNumSubIterations() int32 {
  513. if x != nil && x.NumSubIterations != nil {
  514. return *x.NumSubIterations
  515. }
  516. return Default_TrainerSpec_NumSubIterations
  517. }
  518. func (x *TrainerSpec) GetMaxSentencepieceLength() int32 {
  519. if x != nil && x.MaxSentencepieceLength != nil {
  520. return *x.MaxSentencepieceLength
  521. }
  522. return Default_TrainerSpec_MaxSentencepieceLength
  523. }
  524. func (x *TrainerSpec) GetSplitByUnicodeScript() bool {
  525. if x != nil && x.SplitByUnicodeScript != nil {
  526. return *x.SplitByUnicodeScript
  527. }
  528. return Default_TrainerSpec_SplitByUnicodeScript
  529. }
  530. func (x *TrainerSpec) GetSplitByNumber() bool {
  531. if x != nil && x.SplitByNumber != nil {
  532. return *x.SplitByNumber
  533. }
  534. return Default_TrainerSpec_SplitByNumber
  535. }
  536. func (x *TrainerSpec) GetSplitByWhitespace() bool {
  537. if x != nil && x.SplitByWhitespace != nil {
  538. return *x.SplitByWhitespace
  539. }
  540. return Default_TrainerSpec_SplitByWhitespace
  541. }
  542. func (x *TrainerSpec) GetTreatWhitespaceAsSuffix() bool {
  543. if x != nil && x.TreatWhitespaceAsSuffix != nil {
  544. return *x.TreatWhitespaceAsSuffix
  545. }
  546. return Default_TrainerSpec_TreatWhitespaceAsSuffix
  547. }
  548. func (x *TrainerSpec) GetAllowWhitespaceOnlyPieces() bool {
  549. if x != nil && x.AllowWhitespaceOnlyPieces != nil {
  550. return *x.AllowWhitespaceOnlyPieces
  551. }
  552. return Default_TrainerSpec_AllowWhitespaceOnlyPieces
  553. }
  554. func (x *TrainerSpec) GetSplitDigits() bool {
  555. if x != nil && x.SplitDigits != nil {
  556. return *x.SplitDigits
  557. }
  558. return Default_TrainerSpec_SplitDigits
  559. }
  560. func (x *TrainerSpec) GetPretokenizationDelimiter() string {
  561. if x != nil && x.PretokenizationDelimiter != nil {
  562. return *x.PretokenizationDelimiter
  563. }
  564. return Default_TrainerSpec_PretokenizationDelimiter
  565. }
  566. func (x *TrainerSpec) GetControlSymbols() []string {
  567. if x != nil {
  568. return x.ControlSymbols
  569. }
  570. return nil
  571. }
  572. func (x *TrainerSpec) GetUserDefinedSymbols() []string {
  573. if x != nil {
  574. return x.UserDefinedSymbols
  575. }
  576. return nil
  577. }
  578. func (x *TrainerSpec) GetRequiredChars() string {
  579. if x != nil && x.RequiredChars != nil {
  580. return *x.RequiredChars
  581. }
  582. return ""
  583. }
  584. func (x *TrainerSpec) GetByteFallback() bool {
  585. if x != nil && x.ByteFallback != nil {
  586. return *x.ByteFallback
  587. }
  588. return Default_TrainerSpec_ByteFallback
  589. }
  590. func (x *TrainerSpec) GetVocabularyOutputPieceScore() bool {
  591. if x != nil && x.VocabularyOutputPieceScore != nil {
  592. return *x.VocabularyOutputPieceScore
  593. }
  594. return Default_TrainerSpec_VocabularyOutputPieceScore
  595. }
  596. func (x *TrainerSpec) GetHardVocabLimit() bool {
  597. if x != nil && x.HardVocabLimit != nil {
  598. return *x.HardVocabLimit
  599. }
  600. return Default_TrainerSpec_HardVocabLimit
  601. }
  602. func (x *TrainerSpec) GetUseAllVocab() bool {
  603. if x != nil && x.UseAllVocab != nil {
  604. return *x.UseAllVocab
  605. }
  606. return Default_TrainerSpec_UseAllVocab
  607. }
  608. func (x *TrainerSpec) GetUnkId() int32 {
  609. if x != nil && x.UnkId != nil {
  610. return *x.UnkId
  611. }
  612. return Default_TrainerSpec_UnkId
  613. }
  614. func (x *TrainerSpec) GetBosId() int32 {
  615. if x != nil && x.BosId != nil {
  616. return *x.BosId
  617. }
  618. return Default_TrainerSpec_BosId
  619. }
  620. func (x *TrainerSpec) GetEosId() int32 {
  621. if x != nil && x.EosId != nil {
  622. return *x.EosId
  623. }
  624. return Default_TrainerSpec_EosId
  625. }
  626. func (x *TrainerSpec) GetPadId() int32 {
  627. if x != nil && x.PadId != nil {
  628. return *x.PadId
  629. }
  630. return Default_TrainerSpec_PadId
  631. }
  632. func (x *TrainerSpec) GetUnkPiece() string {
  633. if x != nil && x.UnkPiece != nil {
  634. return *x.UnkPiece
  635. }
  636. return Default_TrainerSpec_UnkPiece
  637. }
  638. func (x *TrainerSpec) GetBosPiece() string {
  639. if x != nil && x.BosPiece != nil {
  640. return *x.BosPiece
  641. }
  642. return Default_TrainerSpec_BosPiece
  643. }
  644. func (x *TrainerSpec) GetEosPiece() string {
  645. if x != nil && x.EosPiece != nil {
  646. return *x.EosPiece
  647. }
  648. return Default_TrainerSpec_EosPiece
  649. }
  650. func (x *TrainerSpec) GetPadPiece() string {
  651. if x != nil && x.PadPiece != nil {
  652. return *x.PadPiece
  653. }
  654. return Default_TrainerSpec_PadPiece
  655. }
  656. func (x *TrainerSpec) GetUnkSurface() string {
  657. if x != nil && x.UnkSurface != nil {
  658. return *x.UnkSurface
  659. }
  660. return Default_TrainerSpec_UnkSurface
  661. }
  662. func (x *TrainerSpec) GetTrainExtremelyLargeCorpus() bool {
  663. if x != nil && x.TrainExtremelyLargeCorpus != nil {
  664. return *x.TrainExtremelyLargeCorpus
  665. }
  666. return Default_TrainerSpec_TrainExtremelyLargeCorpus
  667. }
  668. func (x *TrainerSpec) GetSeedSentencepiecesFile() string {
  669. if x != nil && x.SeedSentencepiecesFile != nil {
  670. return *x.SeedSentencepiecesFile
  671. }
  672. return Default_TrainerSpec_SeedSentencepiecesFile
  673. }
  674. // NormalizerSpec encodes a various parameters for string normalizaiton
  675. type NormalizerSpec struct {
  676. state protoimpl.MessageState
  677. sizeCache protoimpl.SizeCache
  678. unknownFields protoimpl.UnknownFields
  679. extensionFields protoimpl.ExtensionFields
  680. // name of normalization rule.
  681. Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
  682. // Pre-compiled normalization rule created by
  683. // Builder::GetPrecompiledCharsMap() or Builder::CompileCharsMap() method.
  684. // Usually this field is set by Builder::GetNormalizerSpec() method.
  685. PrecompiledCharsmap []byte `protobuf:"bytes,2,opt,name=precompiled_charsmap,json=precompiledCharsmap" json:"precompiled_charsmap,omitempty"`
  686. // Adds dummy whitespace at the beginning of text in order to
  687. // treat "world" in "world" and "hello world" in the same way.
  688. AddDummyPrefix *bool `protobuf:"varint,3,opt,name=add_dummy_prefix,json=addDummyPrefix,def=1" json:"add_dummy_prefix,omitempty"`
  689. // Removes leading, trailing, and duplicate internal whitespace.
  690. RemoveExtraWhitespaces *bool `protobuf:"varint,4,opt,name=remove_extra_whitespaces,json=removeExtraWhitespaces,def=1" json:"remove_extra_whitespaces,omitempty"`
  691. // Replaces whitespace with meta symbol.
  692. // This field must be true to train sentence piece model.
  693. EscapeWhitespaces *bool `protobuf:"varint,5,opt,name=escape_whitespaces,json=escapeWhitespaces,def=1" json:"escape_whitespaces,omitempty"`
  694. // Custom normalization rule file in TSV format.
  695. // https://github.com/google/sentencepiece/blob/master/doc/normalization.md
  696. // This field is only used in SentencePieceTrainer::Train() method, which
  697. // compiles the rule into the binary rule stored in `precompiled_charsmap`.
  698. NormalizationRuleTsv *string `protobuf:"bytes,6,opt,name=normalization_rule_tsv,json=normalizationRuleTsv" json:"normalization_rule_tsv,omitempty"`
  699. }
  700. // Default values for NormalizerSpec fields.
  701. const (
  702. Default_NormalizerSpec_AddDummyPrefix = bool(true)
  703. Default_NormalizerSpec_RemoveExtraWhitespaces = bool(true)
  704. Default_NormalizerSpec_EscapeWhitespaces = bool(true)
  705. )
  706. func (x *NormalizerSpec) Reset() {
  707. *x = NormalizerSpec{}
  708. if protoimpl.UnsafeEnabled {
  709. mi := &file_sentencepiece_model_proto_msgTypes[1]
  710. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  711. ms.StoreMessageInfo(mi)
  712. }
  713. }
  714. func (x *NormalizerSpec) String() string {
  715. return protoimpl.X.MessageStringOf(x)
  716. }
  717. func (*NormalizerSpec) ProtoMessage() {}
  718. func (x *NormalizerSpec) ProtoReflect() protoreflect.Message {
  719. mi := &file_sentencepiece_model_proto_msgTypes[1]
  720. if protoimpl.UnsafeEnabled && x != nil {
  721. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  722. if ms.LoadMessageInfo() == nil {
  723. ms.StoreMessageInfo(mi)
  724. }
  725. return ms
  726. }
  727. return mi.MessageOf(x)
  728. }
  729. // Deprecated: Use NormalizerSpec.ProtoReflect.Descriptor instead.
  730. func (*NormalizerSpec) Descriptor() ([]byte, []int) {
  731. return file_sentencepiece_model_proto_rawDescGZIP(), []int{1}
  732. }
  733. func (x *NormalizerSpec) GetName() string {
  734. if x != nil && x.Name != nil {
  735. return *x.Name
  736. }
  737. return ""
  738. }
  739. func (x *NormalizerSpec) GetPrecompiledCharsmap() []byte {
  740. if x != nil {
  741. return x.PrecompiledCharsmap
  742. }
  743. return nil
  744. }
  745. func (x *NormalizerSpec) GetAddDummyPrefix() bool {
  746. if x != nil && x.AddDummyPrefix != nil {
  747. return *x.AddDummyPrefix
  748. }
  749. return Default_NormalizerSpec_AddDummyPrefix
  750. }
  751. func (x *NormalizerSpec) GetRemoveExtraWhitespaces() bool {
  752. if x != nil && x.RemoveExtraWhitespaces != nil {
  753. return *x.RemoveExtraWhitespaces
  754. }
  755. return Default_NormalizerSpec_RemoveExtraWhitespaces
  756. }
  757. func (x *NormalizerSpec) GetEscapeWhitespaces() bool {
  758. if x != nil && x.EscapeWhitespaces != nil {
  759. return *x.EscapeWhitespaces
  760. }
  761. return Default_NormalizerSpec_EscapeWhitespaces
  762. }
  763. func (x *NormalizerSpec) GetNormalizationRuleTsv() string {
  764. if x != nil && x.NormalizationRuleTsv != nil {
  765. return *x.NormalizationRuleTsv
  766. }
  767. return ""
  768. }
  769. // Proto to store samples for self-testing.
  770. type SelfTestData struct {
  771. state protoimpl.MessageState
  772. sizeCache protoimpl.SizeCache
  773. unknownFields protoimpl.UnknownFields
  774. extensionFields protoimpl.ExtensionFields
  775. Samples []*SelfTestData_Sample `protobuf:"bytes,1,rep,name=samples" json:"samples,omitempty"`
  776. }
  777. func (x *SelfTestData) Reset() {
  778. *x = SelfTestData{}
  779. if protoimpl.UnsafeEnabled {
  780. mi := &file_sentencepiece_model_proto_msgTypes[2]
  781. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  782. ms.StoreMessageInfo(mi)
  783. }
  784. }
  785. func (x *SelfTestData) String() string {
  786. return protoimpl.X.MessageStringOf(x)
  787. }
  788. func (*SelfTestData) ProtoMessage() {}
  789. func (x *SelfTestData) ProtoReflect() protoreflect.Message {
  790. mi := &file_sentencepiece_model_proto_msgTypes[2]
  791. if protoimpl.UnsafeEnabled && x != nil {
  792. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  793. if ms.LoadMessageInfo() == nil {
  794. ms.StoreMessageInfo(mi)
  795. }
  796. return ms
  797. }
  798. return mi.MessageOf(x)
  799. }
  800. // Deprecated: Use SelfTestData.ProtoReflect.Descriptor instead.
  801. func (*SelfTestData) Descriptor() ([]byte, []int) {
  802. return file_sentencepiece_model_proto_rawDescGZIP(), []int{2}
  803. }
  804. func (x *SelfTestData) GetSamples() []*SelfTestData_Sample {
  805. if x != nil {
  806. return x.Samples
  807. }
  808. return nil
  809. }
  810. // ModelProto stores model parameters.
  811. // SentencePieceProcessor is supposed to be self-contained.
  812. // All settings/parameters which may change the behavior must be encoded
  813. // in ModelProto.
  814. type ModelProto struct {
  815. state protoimpl.MessageState
  816. sizeCache protoimpl.SizeCache
  817. unknownFields protoimpl.UnknownFields
  818. extensionFields protoimpl.ExtensionFields
  819. // Sentence pieces with scores.
  820. Pieces []*ModelProto_SentencePiece `protobuf:"bytes,1,rep,name=pieces" json:"pieces,omitempty"`
  821. // Spec used to generate this model file.
  822. TrainerSpec *TrainerSpec `protobuf:"bytes,2,opt,name=trainer_spec,json=trainerSpec" json:"trainer_spec,omitempty"`
  823. // Spec for text normalization.
  824. NormalizerSpec *NormalizerSpec `protobuf:"bytes,3,opt,name=normalizer_spec,json=normalizerSpec" json:"normalizer_spec,omitempty"`
  825. // Stores sample input and its expected segmentation to verify the model.
  826. SelfTestData *SelfTestData `protobuf:"bytes,4,opt,name=self_test_data,json=selfTestData" json:"self_test_data,omitempty"`
  827. // Spec for text de-normalization.
  828. DenormalizerSpec *NormalizerSpec `protobuf:"bytes,5,opt,name=denormalizer_spec,json=denormalizerSpec" json:"denormalizer_spec,omitempty"`
  829. }
  830. func (x *ModelProto) Reset() {
  831. *x = ModelProto{}
  832. if protoimpl.UnsafeEnabled {
  833. mi := &file_sentencepiece_model_proto_msgTypes[3]
  834. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  835. ms.StoreMessageInfo(mi)
  836. }
  837. }
  838. func (x *ModelProto) String() string {
  839. return protoimpl.X.MessageStringOf(x)
  840. }
  841. func (*ModelProto) ProtoMessage() {}
  842. func (x *ModelProto) ProtoReflect() protoreflect.Message {
  843. mi := &file_sentencepiece_model_proto_msgTypes[3]
  844. if protoimpl.UnsafeEnabled && x != nil {
  845. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  846. if ms.LoadMessageInfo() == nil {
  847. ms.StoreMessageInfo(mi)
  848. }
  849. return ms
  850. }
  851. return mi.MessageOf(x)
  852. }
  853. // Deprecated: Use ModelProto.ProtoReflect.Descriptor instead.
  854. func (*ModelProto) Descriptor() ([]byte, []int) {
  855. return file_sentencepiece_model_proto_rawDescGZIP(), []int{3}
  856. }
  857. func (x *ModelProto) GetPieces() []*ModelProto_SentencePiece {
  858. if x != nil {
  859. return x.Pieces
  860. }
  861. return nil
  862. }
  863. func (x *ModelProto) GetTrainerSpec() *TrainerSpec {
  864. if x != nil {
  865. return x.TrainerSpec
  866. }
  867. return nil
  868. }
  869. func (x *ModelProto) GetNormalizerSpec() *NormalizerSpec {
  870. if x != nil {
  871. return x.NormalizerSpec
  872. }
  873. return nil
  874. }
  875. func (x *ModelProto) GetSelfTestData() *SelfTestData {
  876. if x != nil {
  877. return x.SelfTestData
  878. }
  879. return nil
  880. }
  881. func (x *ModelProto) GetDenormalizerSpec() *NormalizerSpec {
  882. if x != nil {
  883. return x.DenormalizerSpec
  884. }
  885. return nil
  886. }
  887. type SelfTestData_Sample struct {
  888. state protoimpl.MessageState
  889. sizeCache protoimpl.SizeCache
  890. unknownFields protoimpl.UnknownFields
  891. Input *string `protobuf:"bytes,1,opt,name=input" json:"input,omitempty"`
  892. Expected *string `protobuf:"bytes,2,opt,name=expected" json:"expected,omitempty"`
  893. }
  894. func (x *SelfTestData_Sample) Reset() {
  895. *x = SelfTestData_Sample{}
  896. if protoimpl.UnsafeEnabled {
  897. mi := &file_sentencepiece_model_proto_msgTypes[4]
  898. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  899. ms.StoreMessageInfo(mi)
  900. }
  901. }
  902. func (x *SelfTestData_Sample) String() string {
  903. return protoimpl.X.MessageStringOf(x)
  904. }
  905. func (*SelfTestData_Sample) ProtoMessage() {}
  906. func (x *SelfTestData_Sample) ProtoReflect() protoreflect.Message {
  907. mi := &file_sentencepiece_model_proto_msgTypes[4]
  908. if protoimpl.UnsafeEnabled && x != nil {
  909. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  910. if ms.LoadMessageInfo() == nil {
  911. ms.StoreMessageInfo(mi)
  912. }
  913. return ms
  914. }
  915. return mi.MessageOf(x)
  916. }
  917. // Deprecated: Use SelfTestData_Sample.ProtoReflect.Descriptor instead.
  918. func (*SelfTestData_Sample) Descriptor() ([]byte, []int) {
  919. return file_sentencepiece_model_proto_rawDescGZIP(), []int{2, 0}
  920. }
  921. func (x *SelfTestData_Sample) GetInput() string {
  922. if x != nil && x.Input != nil {
  923. return *x.Input
  924. }
  925. return ""
  926. }
  927. func (x *SelfTestData_Sample) GetExpected() string {
  928. if x != nil && x.Expected != nil {
  929. return *x.Expected
  930. }
  931. return ""
  932. }
  933. type ModelProto_SentencePiece struct {
  934. state protoimpl.MessageState
  935. sizeCache protoimpl.SizeCache
  936. unknownFields protoimpl.UnknownFields
  937. extensionFields protoimpl.ExtensionFields
  938. Piece *string `protobuf:"bytes,1,opt,name=piece" json:"piece,omitempty"` // piece must not be empty.
  939. Score *float32 `protobuf:"fixed32,2,opt,name=score" json:"score,omitempty"`
  940. Type *ModelProto_SentencePiece_Type `protobuf:"varint,3,opt,name=type,enum=sentencepiece.ModelProto_SentencePiece_Type,def=1" json:"type,omitempty"`
  941. }
  942. // Default values for ModelProto_SentencePiece fields.
  943. const (
  944. Default_ModelProto_SentencePiece_Type = ModelProto_SentencePiece_NORMAL
  945. )
  946. func (x *ModelProto_SentencePiece) Reset() {
  947. *x = ModelProto_SentencePiece{}
  948. if protoimpl.UnsafeEnabled {
  949. mi := &file_sentencepiece_model_proto_msgTypes[5]
  950. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  951. ms.StoreMessageInfo(mi)
  952. }
  953. }
  954. func (x *ModelProto_SentencePiece) String() string {
  955. return protoimpl.X.MessageStringOf(x)
  956. }
  957. func (*ModelProto_SentencePiece) ProtoMessage() {}
  958. func (x *ModelProto_SentencePiece) ProtoReflect() protoreflect.Message {
  959. mi := &file_sentencepiece_model_proto_msgTypes[5]
  960. if protoimpl.UnsafeEnabled && x != nil {
  961. ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
  962. if ms.LoadMessageInfo() == nil {
  963. ms.StoreMessageInfo(mi)
  964. }
  965. return ms
  966. }
  967. return mi.MessageOf(x)
  968. }
  969. // Deprecated: Use ModelProto_SentencePiece.ProtoReflect.Descriptor instead.
  970. func (*ModelProto_SentencePiece) Descriptor() ([]byte, []int) {
  971. return file_sentencepiece_model_proto_rawDescGZIP(), []int{3, 0}
  972. }
  973. func (x *ModelProto_SentencePiece) GetPiece() string {
  974. if x != nil && x.Piece != nil {
  975. return *x.Piece
  976. }
  977. return ""
  978. }
  979. func (x *ModelProto_SentencePiece) GetScore() float32 {
  980. if x != nil && x.Score != nil {
  981. return *x.Score
  982. }
  983. return 0
  984. }
  985. func (x *ModelProto_SentencePiece) GetType() ModelProto_SentencePiece_Type {
  986. if x != nil && x.Type != nil {
  987. return *x.Type
  988. }
  989. return Default_ModelProto_SentencePiece_Type
  990. }
  991. var File_sentencepiece_model_proto protoreflect.FileDescriptor
  992. var file_sentencepiece_model_proto_rawDesc = []byte{
  993. 0x0a, 0x19, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f,
  994. 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x73, 0x65, 0x6e,
  995. 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x22, 0xc6, 0x12, 0x0a, 0x0b, 0x54,
  996. 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e,
  997. 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74,
  998. 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74,
  999. 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, 0x6f, 0x72,
  1000. 0x6d, 0x61, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x70, 0x72, 0x65,
  1001. 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x6f, 0x64, 0x65, 0x6c,
  1002. 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x4c, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f,
  1003. 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x73, 0x65, 0x6e,
  1004. 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x54, 0x72, 0x61, 0x69, 0x6e,
  1005. 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65,
  1006. 0x3a, 0x07, 0x55, 0x4e, 0x49, 0x47, 0x52, 0x41, 0x4d, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c,
  1007. 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0a, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x5f, 0x73, 0x69,
  1008. 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x04, 0x38, 0x30, 0x30, 0x30, 0x52, 0x09,
  1009. 0x76, 0x6f, 0x63, 0x61, 0x62, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x63, 0x63,
  1010. 0x65, 0x70, 0x74, 0x5f, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x03,
  1011. 0x28, 0x09, 0x52, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x4c, 0x61, 0x6e, 0x67, 0x75, 0x61,
  1012. 0x67, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x73, 0x65, 0x6c, 0x66, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f,
  1013. 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28,
  1014. 0x05, 0x3a, 0x01, 0x30, 0x52, 0x12, 0x73, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x53, 0x61,
  1015. 0x6d, 0x70, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x45, 0x0a, 0x1b, 0x65, 0x6e, 0x61, 0x62,
  1016. 0x6c, 0x65, 0x5f, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f,
  1017. 0x70, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x18, 0x32, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66,
  1018. 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x44, 0x69, 0x66, 0x66,
  1019. 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x12,
  1020. 0x4a, 0x0a, 0x20, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f,
  1021. 0x70, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x5f, 0x6e, 0x6f, 0x69, 0x73, 0x65, 0x5f, 0x6c, 0x65,
  1022. 0x76, 0x65, 0x6c, 0x18, 0x33, 0x20, 0x01, 0x28, 0x02, 0x3a, 0x01, 0x30, 0x52, 0x1d, 0x64, 0x69,
  1023. 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63,
  1024. 0x79, 0x4e, 0x6f, 0x69, 0x73, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x58, 0x0a, 0x27, 0x64,
  1025. 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x70, 0x72, 0x69, 0x76,
  1026. 0x61, 0x63, 0x79, 0x5f, 0x63, 0x6c, 0x69, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x68, 0x72,
  1027. 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x34, 0x20, 0x01, 0x28, 0x04, 0x3a, 0x01, 0x30, 0x52,
  1028. 0x24, 0x64, 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x72, 0x69,
  1029. 0x76, 0x61, 0x63, 0x79, 0x43, 0x6c, 0x69, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x54, 0x68, 0x72, 0x65,
  1030. 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x35, 0x0a, 0x12, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74,
  1031. 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28,
  1032. 0x02, 0x3a, 0x06, 0x30, 0x2e, 0x39, 0x39, 0x39, 0x35, 0x52, 0x11, 0x63, 0x68, 0x61, 0x72, 0x61,
  1033. 0x63, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x12, 0x31, 0x0a, 0x13,
  1034. 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73,
  1035. 0x69, 0x7a, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x3a, 0x01, 0x30, 0x52, 0x11, 0x69, 0x6e,
  1036. 0x70, 0x75, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12,
  1037. 0x3a, 0x0a, 0x16, 0x73, 0x68, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74,
  1038. 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x3a,
  1039. 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x14, 0x73, 0x68, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x49, 0x6e,
  1040. 0x70, 0x75, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x14, 0x6d,
  1041. 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73,
  1042. 0x69, 0x7a, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x42, 0x02, 0x18, 0x01, 0x52, 0x12, 0x6d,
  1043. 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a,
  1044. 0x65, 0x12, 0x38, 0x0a, 0x16, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65,
  1045. 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28,
  1046. 0x05, 0x42, 0x02, 0x18, 0x01, 0x52, 0x14, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53,
  1047. 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x3f, 0x0a, 0x17, 0x73,
  1048. 0x65, 0x65, 0x64, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
  1049. 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x07, 0x31, 0x30,
  1050. 0x30, 0x30, 0x30, 0x30, 0x30, 0x52, 0x15, 0x73, 0x65, 0x65, 0x64, 0x53, 0x65, 0x6e, 0x74, 0x65,
  1051. 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2f, 0x0a, 0x10,
  1052. 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72,
  1053. 0x18, 0x0f, 0x20, 0x01, 0x28, 0x02, 0x3a, 0x04, 0x30, 0x2e, 0x37, 0x35, 0x52, 0x0f, 0x73, 0x68,
  1054. 0x72, 0x69, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x34, 0x0a,
  1055. 0x13, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6c, 0x65,
  1056. 0x6e, 0x67, 0x74, 0x68, 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x04, 0x34, 0x31, 0x39, 0x32,
  1057. 0x52, 0x11, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x4c, 0x65, 0x6e,
  1058. 0x67, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61,
  1059. 0x64, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x31, 0x36, 0x52, 0x0a, 0x6e, 0x75,
  1060. 0x6d, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x2f, 0x0a, 0x12, 0x6e, 0x75, 0x6d, 0x5f,
  1061. 0x73, 0x75, 0x62, 0x5f, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x11,
  1062. 0x20, 0x01, 0x28, 0x05, 0x3a, 0x01, 0x32, 0x52, 0x10, 0x6e, 0x75, 0x6d, 0x53, 0x75, 0x62, 0x49,
  1063. 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3c, 0x0a, 0x18, 0x6d, 0x61, 0x78,
  1064. 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f, 0x6c,
  1065. 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x31, 0x36, 0x52,
  1066. 0x16, 0x6d, 0x61, 0x78, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
  1067. 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x3b, 0x0a, 0x17, 0x73, 0x70, 0x6c, 0x69, 0x74,
  1068. 0x5f, 0x62, 0x79, 0x5f, 0x75, 0x6e, 0x69, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x73, 0x63, 0x72, 0x69,
  1069. 0x70, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x14,
  1070. 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x55, 0x6e, 0x69, 0x63, 0x6f, 0x64, 0x65, 0x53, 0x63,
  1071. 0x72, 0x69, 0x70, 0x74, 0x12, 0x2c, 0x0a, 0x0f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x62, 0x79,
  1072. 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74,
  1073. 0x72, 0x75, 0x65, 0x52, 0x0d, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x4e, 0x75, 0x6d, 0x62,
  1074. 0x65, 0x72, 0x12, 0x34, 0x0a, 0x13, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x77,
  1075. 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x3a,
  1076. 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x11, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x42, 0x79, 0x57, 0x68,
  1077. 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x42, 0x0a, 0x1a, 0x74, 0x72, 0x65, 0x61,
  1078. 0x74, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x61, 0x73, 0x5f,
  1079. 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61,
  1080. 0x6c, 0x73, 0x65, 0x52, 0x17, 0x74, 0x72, 0x65, 0x61, 0x74, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73,
  1081. 0x70, 0x61, 0x63, 0x65, 0x41, 0x73, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x12, 0x46, 0x0a, 0x1c,
  1082. 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65,
  1083. 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x18, 0x1a, 0x20, 0x01,
  1084. 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77,
  1085. 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x50, 0x69,
  1086. 0x65, 0x63, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x0c, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x64, 0x69,
  1087. 0x67, 0x69, 0x74, 0x73, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73,
  1088. 0x65, 0x52, 0x0b, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x44, 0x69, 0x67, 0x69, 0x74, 0x73, 0x12, 0x3d,
  1089. 0x0a, 0x19, 0x70, 0x72, 0x65, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f,
  1090. 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x35, 0x20, 0x01, 0x28,
  1091. 0x09, 0x3a, 0x00, 0x52, 0x18, 0x70, 0x72, 0x65, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x69, 0x7a, 0x61,
  1092. 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x27, 0x0a,
  1093. 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73,
  1094. 0x18, 0x1e, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53,
  1095. 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x64,
  1096. 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x18, 0x1f,
  1097. 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x75, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65,
  1098. 0x64, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x71, 0x75,
  1099. 0x69, 0x72, 0x65, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x73, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09,
  1100. 0x52, 0x0d, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x43, 0x68, 0x61, 0x72, 0x73, 0x12,
  1101. 0x2a, 0x0a, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b,
  1102. 0x18, 0x23, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0c, 0x62,
  1103. 0x79, 0x74, 0x65, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x47, 0x0a, 0x1d, 0x76,
  1104. 0x6f, 0x63, 0x61, 0x62, 0x75, 0x6c, 0x61, 0x72, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74,
  1105. 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x20, 0x20, 0x01,
  1106. 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x1a, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x75,
  1107. 0x6c, 0x61, 0x72, 0x79, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x50, 0x69, 0x65, 0x63, 0x65, 0x53,
  1108. 0x63, 0x6f, 0x72, 0x65, 0x12, 0x2e, 0x0a, 0x10, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x76, 0x6f, 0x63,
  1109. 0x61, 0x62, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x21, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04,
  1110. 0x74, 0x72, 0x75, 0x65, 0x52, 0x0e, 0x68, 0x61, 0x72, 0x64, 0x56, 0x6f, 0x63, 0x61, 0x62, 0x4c,
  1111. 0x69, 0x6d, 0x69, 0x74, 0x12, 0x29, 0x0a, 0x0d, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x6c, 0x5f,
  1112. 0x76, 0x6f, 0x63, 0x61, 0x62, 0x18, 0x22, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c,
  1113. 0x73, 0x65, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x41, 0x6c, 0x6c, 0x56, 0x6f, 0x63, 0x61, 0x62, 0x12,
  1114. 0x18, 0x0a, 0x06, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x28, 0x20, 0x01, 0x28, 0x05, 0x3a,
  1115. 0x01, 0x30, 0x52, 0x05, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x06, 0x62, 0x6f, 0x73,
  1116. 0x5f, 0x69, 0x64, 0x18, 0x29, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x01, 0x31, 0x52, 0x05, 0x62, 0x6f,
  1117. 0x73, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x06, 0x65, 0x6f, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x2a, 0x20,
  1118. 0x01, 0x28, 0x05, 0x3a, 0x01, 0x32, 0x52, 0x05, 0x65, 0x6f, 0x73, 0x49, 0x64, 0x12, 0x19, 0x0a,
  1119. 0x06, 0x70, 0x61, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, 0x2d,
  1120. 0x31, 0x52, 0x05, 0x70, 0x61, 0x64, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x09, 0x75, 0x6e, 0x6b, 0x5f,
  1121. 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x3c, 0x75, 0x6e,
  1122. 0x6b, 0x3e, 0x52, 0x08, 0x75, 0x6e, 0x6b, 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x09,
  1123. 0x62, 0x6f, 0x73, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2e, 0x20, 0x01, 0x28, 0x09, 0x3a,
  1124. 0x03, 0x3c, 0x73, 0x3e, 0x52, 0x08, 0x62, 0x6f, 0x73, 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x21,
  1125. 0x0a, 0x09, 0x65, 0x6f, 0x73, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x2f, 0x20, 0x01, 0x28,
  1126. 0x09, 0x3a, 0x04, 0x3c, 0x2f, 0x73, 0x3e, 0x52, 0x08, 0x65, 0x6f, 0x73, 0x50, 0x69, 0x65, 0x63,
  1127. 0x65, 0x12, 0x22, 0x0a, 0x09, 0x70, 0x61, 0x64, 0x5f, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x30,
  1128. 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x3c, 0x70, 0x61, 0x64, 0x3e, 0x52, 0x08, 0x70, 0x61, 0x64,
  1129. 0x50, 0x69, 0x65, 0x63, 0x65, 0x12, 0x26, 0x0a, 0x0b, 0x75, 0x6e, 0x6b, 0x5f, 0x73, 0x75, 0x72,
  1130. 0x66, 0x61, 0x63, 0x65, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x05, 0x20, 0xe2, 0x81, 0x87,
  1131. 0x20, 0x52, 0x0a, 0x75, 0x6e, 0x6b, 0x53, 0x75, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x46, 0x0a,
  1132. 0x1c, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79,
  1133. 0x5f, 0x6c, 0x61, 0x72, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x72, 0x70, 0x75, 0x73, 0x18, 0x31, 0x20,
  1134. 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x19, 0x74, 0x72, 0x61, 0x69,
  1135. 0x6e, 0x45, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79, 0x4c, 0x61, 0x72, 0x67, 0x65, 0x43,
  1136. 0x6f, 0x72, 0x70, 0x75, 0x73, 0x12, 0x3a, 0x0a, 0x18, 0x73, 0x65, 0x65, 0x64, 0x5f, 0x73, 0x65,
  1137. 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x5f, 0x66, 0x69, 0x6c,
  1138. 0x65, 0x18, 0x36, 0x20, 0x01, 0x28, 0x09, 0x3a, 0x00, 0x52, 0x16, 0x73, 0x65, 0x65, 0x64, 0x53,
  1139. 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x46, 0x69, 0x6c,
  1140. 0x65, 0x22, 0x35, 0x0a, 0x09, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b,
  1141. 0x0a, 0x07, 0x55, 0x4e, 0x49, 0x47, 0x52, 0x41, 0x4d, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x42,
  1142. 0x50, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x4f, 0x52, 0x44, 0x10, 0x03, 0x12, 0x08,
  1143. 0x0a, 0x04, 0x43, 0x48, 0x41, 0x52, 0x10, 0x04, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
  1144. 0x80, 0x80, 0x02, 0x22, 0xbd, 0x02, 0x0a, 0x0e, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a,
  1145. 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
  1146. 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x31, 0x0a, 0x14, 0x70, 0x72,
  1147. 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x73, 0x6d,
  1148. 0x61, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x70, 0x72, 0x65, 0x63, 0x6f, 0x6d,
  1149. 0x70, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x68, 0x61, 0x72, 0x73, 0x6d, 0x61, 0x70, 0x12, 0x2e, 0x0a,
  1150. 0x10, 0x61, 0x64, 0x64, 0x5f, 0x64, 0x75, 0x6d, 0x6d, 0x79, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69,
  1151. 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x0e, 0x61,
  1152. 0x64, 0x64, 0x44, 0x75, 0x6d, 0x6d, 0x79, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x3e, 0x0a,
  1153. 0x18, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x77, 0x68,
  1154. 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a,
  1155. 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x78, 0x74,
  1156. 0x72, 0x61, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x33, 0x0a,
  1157. 0x12, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61,
  1158. 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52,
  1159. 0x11, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x57, 0x68, 0x69, 0x74, 0x65, 0x73, 0x70, 0x61, 0x63,
  1160. 0x65, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74,
  1161. 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x5f, 0x74, 0x73, 0x76, 0x18, 0x06, 0x20, 0x01,
  1162. 0x28, 0x09, 0x52, 0x14, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f,
  1163. 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x54, 0x73, 0x76, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
  1164. 0x80, 0x80, 0x02, 0x22, 0x93, 0x01, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74,
  1165. 0x44, 0x61, 0x74, 0x61, 0x12, 0x3c, 0x0a, 0x07, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x18,
  1166. 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65,
  1167. 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x44, 0x61,
  1168. 0x74, 0x61, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x07, 0x73, 0x61, 0x6d, 0x70, 0x6c,
  1169. 0x65, 0x73, 0x1a, 0x3a, 0x0a, 0x06, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05,
  1170. 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70,
  1171. 0x75, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02,
  1172. 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x2a, 0x09,
  1173. 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0xd7, 0x04, 0x0a, 0x0a, 0x4d, 0x6f,
  1174. 0x64, 0x65, 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x3f, 0x0a, 0x06, 0x70, 0x69, 0x65, 0x63,
  1175. 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65,
  1176. 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x72,
  1177. 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x69, 0x65, 0x63,
  1178. 0x65, 0x52, 0x06, 0x70, 0x69, 0x65, 0x63, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x0c, 0x74, 0x72, 0x61,
  1179. 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
  1180. 0x1a, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e,
  1181. 0x54, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0b, 0x74, 0x72, 0x61,
  1182. 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x46, 0x0a, 0x0f, 0x6e, 0x6f, 0x72, 0x6d,
  1183. 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28,
  1184. 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63,
  1185. 0x65, 0x2e, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63,
  1186. 0x52, 0x0e, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63,
  1187. 0x12, 0x41, 0x0a, 0x0e, 0x73, 0x65, 0x6c, 0x66, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x64, 0x61,
  1188. 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65,
  1189. 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73,
  1190. 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0c, 0x73, 0x65, 0x6c, 0x66, 0x54, 0x65, 0x73, 0x74, 0x44,
  1191. 0x61, 0x74, 0x61, 0x12, 0x4a, 0x0a, 0x11, 0x64, 0x65, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69,
  1192. 0x7a, 0x65, 0x72, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d,
  1193. 0x2e, 0x73, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4e,
  1194. 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x10, 0x64,
  1195. 0x65, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x1a,
  1196. 0xe6, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x69, 0x65, 0x63,
  1197. 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x69, 0x65, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
  1198. 0x52, 0x05, 0x70, 0x69, 0x65, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65,
  1199. 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x48, 0x0a,
  1200. 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x73, 0x65,
  1201. 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65,
  1202. 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x6e, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x50,
  1203. 0x69, 0x65, 0x63, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41,
  1204. 0x4c, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x54, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12,
  1205. 0x0a, 0x0a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55,
  1206. 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x54,
  1207. 0x52, 0x4f, 0x4c, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x44, 0x45,
  1208. 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x59, 0x54, 0x45, 0x10,
  1209. 0x06, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x4e, 0x55, 0x53, 0x45, 0x44, 0x10, 0x05, 0x2a, 0x09, 0x08,
  1210. 0xc8, 0x01, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x2a, 0x09, 0x08, 0xc8, 0x01, 0x10, 0x80, 0x80,
  1211. 0x80, 0x80, 0x02, 0x42, 0x13, 0x48, 0x03, 0x5a, 0x0f, 0x2e, 0x2f, 0x73, 0x65, 0x6e, 0x74, 0x65,
  1212. 0x6e, 0x63, 0x65, 0x70, 0x69, 0x65, 0x63, 0x65,
  1213. }
  1214. var (
  1215. file_sentencepiece_model_proto_rawDescOnce sync.Once
  1216. file_sentencepiece_model_proto_rawDescData = file_sentencepiece_model_proto_rawDesc
  1217. )
  1218. func file_sentencepiece_model_proto_rawDescGZIP() []byte {
  1219. file_sentencepiece_model_proto_rawDescOnce.Do(func() {
  1220. file_sentencepiece_model_proto_rawDescData = protoimpl.X.CompressGZIP(file_sentencepiece_model_proto_rawDescData)
  1221. })
  1222. return file_sentencepiece_model_proto_rawDescData
  1223. }
  1224. var file_sentencepiece_model_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
  1225. var file_sentencepiece_model_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
  1226. var file_sentencepiece_model_proto_goTypes = []interface{}{
  1227. (TrainerSpec_ModelType)(0), // 0: sentencepiece.TrainerSpec.ModelType
  1228. (ModelProto_SentencePiece_Type)(0), // 1: sentencepiece.ModelProto.SentencePiece.Type
  1229. (*TrainerSpec)(nil), // 2: sentencepiece.TrainerSpec
  1230. (*NormalizerSpec)(nil), // 3: sentencepiece.NormalizerSpec
  1231. (*SelfTestData)(nil), // 4: sentencepiece.SelfTestData
  1232. (*ModelProto)(nil), // 5: sentencepiece.ModelProto
  1233. (*SelfTestData_Sample)(nil), // 6: sentencepiece.SelfTestData.Sample
  1234. (*ModelProto_SentencePiece)(nil), // 7: sentencepiece.ModelProto.SentencePiece
  1235. }
  1236. var file_sentencepiece_model_proto_depIdxs = []int32{
  1237. 0, // 0: sentencepiece.TrainerSpec.model_type:type_name -> sentencepiece.TrainerSpec.ModelType
  1238. 6, // 1: sentencepiece.SelfTestData.samples:type_name -> sentencepiece.SelfTestData.Sample
  1239. 7, // 2: sentencepiece.ModelProto.pieces:type_name -> sentencepiece.ModelProto.SentencePiece
  1240. 2, // 3: sentencepiece.ModelProto.trainer_spec:type_name -> sentencepiece.TrainerSpec
  1241. 3, // 4: sentencepiece.ModelProto.normalizer_spec:type_name -> sentencepiece.NormalizerSpec
  1242. 4, // 5: sentencepiece.ModelProto.self_test_data:type_name -> sentencepiece.SelfTestData
  1243. 3, // 6: sentencepiece.ModelProto.denormalizer_spec:type_name -> sentencepiece.NormalizerSpec
  1244. 1, // 7: sentencepiece.ModelProto.SentencePiece.type:type_name -> sentencepiece.ModelProto.SentencePiece.Type
  1245. 8, // [8:8] is the sub-list for method output_type
  1246. 8, // [8:8] is the sub-list for method input_type
  1247. 8, // [8:8] is the sub-list for extension type_name
  1248. 8, // [8:8] is the sub-list for extension extendee
  1249. 0, // [0:8] is the sub-list for field type_name
  1250. }
  1251. func init() { file_sentencepiece_model_proto_init() }
  1252. func file_sentencepiece_model_proto_init() {
  1253. if File_sentencepiece_model_proto != nil {
  1254. return
  1255. }
  1256. if !protoimpl.UnsafeEnabled {
  1257. file_sentencepiece_model_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
  1258. switch v := v.(*TrainerSpec); i {
  1259. case 0:
  1260. return &v.state
  1261. case 1:
  1262. return &v.sizeCache
  1263. case 2:
  1264. return &v.unknownFields
  1265. case 3:
  1266. return &v.extensionFields
  1267. default:
  1268. return nil
  1269. }
  1270. }
  1271. file_sentencepiece_model_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
  1272. switch v := v.(*NormalizerSpec); i {
  1273. case 0:
  1274. return &v.state
  1275. case 1:
  1276. return &v.sizeCache
  1277. case 2:
  1278. return &v.unknownFields
  1279. case 3:
  1280. return &v.extensionFields
  1281. default:
  1282. return nil
  1283. }
  1284. }
  1285. file_sentencepiece_model_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
  1286. switch v := v.(*SelfTestData); i {
  1287. case 0:
  1288. return &v.state
  1289. case 1:
  1290. return &v.sizeCache
  1291. case 2:
  1292. return &v.unknownFields
  1293. case 3:
  1294. return &v.extensionFields
  1295. default:
  1296. return nil
  1297. }
  1298. }
  1299. file_sentencepiece_model_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
  1300. switch v := v.(*ModelProto); i {
  1301. case 0:
  1302. return &v.state
  1303. case 1:
  1304. return &v.sizeCache
  1305. case 2:
  1306. return &v.unknownFields
  1307. case 3:
  1308. return &v.extensionFields
  1309. default:
  1310. return nil
  1311. }
  1312. }
  1313. file_sentencepiece_model_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
  1314. switch v := v.(*SelfTestData_Sample); i {
  1315. case 0:
  1316. return &v.state
  1317. case 1:
  1318. return &v.sizeCache
  1319. case 2:
  1320. return &v.unknownFields
  1321. default:
  1322. return nil
  1323. }
  1324. }
  1325. file_sentencepiece_model_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
  1326. switch v := v.(*ModelProto_SentencePiece); i {
  1327. case 0:
  1328. return &v.state
  1329. case 1:
  1330. return &v.sizeCache
  1331. case 2:
  1332. return &v.unknownFields
  1333. case 3:
  1334. return &v.extensionFields
  1335. default:
  1336. return nil
  1337. }
  1338. }
  1339. }
  1340. type x struct{}
  1341. out := protoimpl.TypeBuilder{
  1342. File: protoimpl.DescBuilder{
  1343. GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
  1344. RawDescriptor: file_sentencepiece_model_proto_rawDesc,
  1345. NumEnums: 2,
  1346. NumMessages: 6,
  1347. NumExtensions: 0,
  1348. NumServices: 0,
  1349. },
  1350. GoTypes: file_sentencepiece_model_proto_goTypes,
  1351. DependencyIndexes: file_sentencepiece_model_proto_depIdxs,
  1352. EnumInfos: file_sentencepiece_model_proto_enumTypes,
  1353. MessageInfos: file_sentencepiece_model_proto_msgTypes,
  1354. }.Build()
  1355. File_sentencepiece_model_proto = out.File
  1356. file_sentencepiece_model_proto_rawDesc = nil
  1357. file_sentencepiece_model_proto_goTypes = nil
  1358. file_sentencepiece_model_proto_depIdxs = nil
  1359. }