Repository Reading Site
tokenizer.json
ml-platform/examples/20-llm-training/03-base-model/tokenizer.json
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 151643,
"content": "<|im_start|>",
"special": true
},
{
"id": 151644,
"content": "<|im_end|>",
"special": true
},
{
"id": 151645,
"content": "<|endoftext|>",
"special": true
}
],
"model": {
"type": "BPE",
"vocab": {
"<|im_start|>": 151643,
"<|im_end|>": 151644,
"<|endoftext|>": 151645,
"Kubernetes": 30001,
"Pod": 30002,
"LoRA": 30003
},
"merges": []
}
}