233 lines
4.8 KiB
JSON
233 lines
4.8 KiB
JSON
{
|
|
"version": "1.0",
|
|
"truncation": null,
|
|
"padding": null,
|
|
"added_tokens": [],
|
|
"normalizer": {
|
|
"type": "Replace",
|
|
"pattern": {
|
|
"Regex": "[^ !\"(),./12345:;?AIOQRSTWYabcdefhijklmnopqrstuvwxyz\u00e6\u00e7\u00f0\u00f8\u014b\u0153\u0250\u0251\u0252\u0254\u0255\u0256\u0259\u025b\u025c\u025f\u0261\u0263\u0268\u026a\u026f\u0270\u0272\u0273\u0274\u0278\u0279\u027d\u027e\u0281\u0282\u0283\u0288\u028a\u028b\u028c\u028e\u0292\u0294\u029d\u02a3\u02a4\u02a5\u02a6\u02a7\u02a8\u02b0\u02b2\u02c8\u02cc\u02d0\u0303\u03b2\u03b8\u03c7\u1d4a\u1d5d\u1d7b\u2014\u201c\u201d\u2026\u3105\u3106\u3107\u3108\u3109\u310a\u310b\u310c\u310d\u310e\u310f\u3110\u3111\u3112\u3113\u3114\u3115\u3116\u3117\u3118\u3119\u311a\u311b\u311c\u311d\u311e\u311f\u3120\u3121\u3122\u3123\u3124\u3125\u3126\u3127\u3128\u3129\u312d\u4e07\u4e2d\u4e3a\u4e91\u5143\u5341\u538b\u53c8\u5916\u5e94\u6211\u6587\u6708\u738b\u74ee\u7528\u7a75\u8981\u8a00\u9633\u9634]"
|
|
},
|
|
"content": ""
|
|
},
|
|
"pre_tokenizer": {
|
|
"type": "Split",
|
|
"pattern": {
|
|
"Regex": ""
|
|
},
|
|
"behavior": "Isolated",
|
|
"invert": false
|
|
},
|
|
"post_processor": {
|
|
"type": "TemplateProcessing",
|
|
"single": [
|
|
{
|
|
"SpecialToken": {
|
|
"id": "$",
|
|
"type_id": 0
|
|
}
|
|
},
|
|
{
|
|
"Sequence": {
|
|
"id": "A",
|
|
"type_id": 0
|
|
}
|
|
},
|
|
{
|
|
"SpecialToken": {
|
|
"id": "$",
|
|
"type_id": 0
|
|
}
|
|
}
|
|
],
|
|
"special_tokens": {
|
|
"$": {
|
|
"id": "$",
|
|
"ids": [
|
|
0
|
|
],
|
|
"tokens": [
|
|
"$"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"decoder": null,
|
|
"model": {
|
|
"vocab": {
|
|
"$": 0,
|
|
";": 1,
|
|
":": 2,
|
|
",": 3,
|
|
".": 4,
|
|
"!": 5,
|
|
"?": 6,
|
|
"/": 7,
|
|
"\u2014": 9,
|
|
"\u2026": 10,
|
|
"\"": 11,
|
|
"(": 12,
|
|
")": 13,
|
|
"\u201c": 14,
|
|
"\u201d": 15,
|
|
" ": 16,
|
|
"\u0303": 17,
|
|
"\u02a3": 18,
|
|
"\u02a5": 19,
|
|
"\u02a6": 20,
|
|
"\u02a8": 21,
|
|
"\u1d5d": 22,
|
|
"\u3113": 23,
|
|
"A": 24,
|
|
"I": 25,
|
|
"\u3105": 30,
|
|
"O": 31,
|
|
"\u3106": 32,
|
|
"Q": 33,
|
|
"R": 34,
|
|
"S": 35,
|
|
"T": 36,
|
|
"\u3107": 37,
|
|
"\u3108": 38,
|
|
"W": 39,
|
|
"\u3109": 40,
|
|
"Y": 41,
|
|
"\u1d4a": 42,
|
|
"a": 43,
|
|
"b": 44,
|
|
"c": 45,
|
|
"d": 46,
|
|
"e": 47,
|
|
"f": 48,
|
|
"\u310a": 49,
|
|
"h": 50,
|
|
"i": 51,
|
|
"j": 52,
|
|
"k": 53,
|
|
"l": 54,
|
|
"m": 55,
|
|
"n": 56,
|
|
"o": 57,
|
|
"p": 58,
|
|
"q": 59,
|
|
"r": 60,
|
|
"s": 61,
|
|
"t": 62,
|
|
"u": 63,
|
|
"v": 64,
|
|
"w": 65,
|
|
"x": 66,
|
|
"y": 67,
|
|
"z": 68,
|
|
"\u0251": 69,
|
|
"\u0250": 70,
|
|
"\u0252": 71,
|
|
"\u00e6": 72,
|
|
"\u310b": 73,
|
|
"\u310c": 74,
|
|
"\u03b2": 75,
|
|
"\u0254": 76,
|
|
"\u0255": 77,
|
|
"\u00e7": 78,
|
|
"\u310d": 79,
|
|
"\u0256": 80,
|
|
"\u00f0": 81,
|
|
"\u02a4": 82,
|
|
"\u0259": 83,
|
|
"\u310e": 84,
|
|
"\u3126": 85,
|
|
"\u025b": 86,
|
|
"\u025c": 87,
|
|
"\u310f": 88,
|
|
"\u3110": 89,
|
|
"\u025f": 90,
|
|
"\u3111": 91,
|
|
"\u0261": 92,
|
|
"\u3112": 93,
|
|
"\u3114": 94,
|
|
"\u3115": 95,
|
|
"\u3117": 96,
|
|
"\u3118": 97,
|
|
"\u3119": 98,
|
|
"\u6708": 99,
|
|
"\u311a": 100,
|
|
"\u0268": 101,
|
|
"\u026a": 102,
|
|
"\u029d": 103,
|
|
"\u311b": 104,
|
|
"\u311d": 105,
|
|
"\u311e": 106,
|
|
"\u311f": 107,
|
|
"\u3120": 108,
|
|
"\u3121": 109,
|
|
"\u026f": 110,
|
|
"\u0270": 111,
|
|
"\u014b": 112,
|
|
"\u0273": 113,
|
|
"\u0272": 114,
|
|
"\u0274": 115,
|
|
"\u00f8": 116,
|
|
"\u3122": 117,
|
|
"\u0278": 118,
|
|
"\u03b8": 119,
|
|
"\u0153": 120,
|
|
"\u3123": 121,
|
|
"\u3124": 122,
|
|
"\u0279": 123,
|
|
"\u3125": 124,
|
|
"\u027e": 125,
|
|
"\u3116": 126,
|
|
"\u3127": 127,
|
|
"\u0281": 128,
|
|
"\u027d": 129,
|
|
"\u0282": 130,
|
|
"\u0283": 131,
|
|
"\u0288": 132,
|
|
"\u02a7": 133,
|
|
"\u3128": 134,
|
|
"\u028a": 135,
|
|
"\u028b": 136,
|
|
"\u3129": 137,
|
|
"\u028c": 138,
|
|
"\u0263": 139,
|
|
"\u311c": 140,
|
|
"\u312d": 141,
|
|
"\u03c7": 142,
|
|
"\u028e": 143,
|
|
"\u5341": 144,
|
|
"\u538b": 145,
|
|
"\u8a00": 146,
|
|
"\u0292": 147,
|
|
"\u0294": 148,
|
|
"\u9633": 149,
|
|
"\u8981": 150,
|
|
"\u9634": 151,
|
|
"\u5e94": 152,
|
|
"\u7528": 153,
|
|
"\u53c8": 154,
|
|
"\u4e2d": 155,
|
|
"\u02c8": 156,
|
|
"\u02cc": 157,
|
|
"\u02d0": 158,
|
|
"\u7a75": 159,
|
|
"\u5916": 160,
|
|
"\u4e07": 161,
|
|
"\u02b0": 162,
|
|
"\u738b": 163,
|
|
"\u02b2": 164,
|
|
"\u4e3a": 165,
|
|
"\u6587": 166,
|
|
"\u74ee": 167,
|
|
"\u6211": 168,
|
|
"3": 169,
|
|
"5": 170,
|
|
"1": 171,
|
|
"2": 172,
|
|
"4": 173,
|
|
"\u5143": 175,
|
|
"\u4e91": 176,
|
|
"\u1d7b": 177
|
|
}
|
|
}
|
|
}
|