K8s Lab 把当前仓库文档整理成一个可阅读的网页站点

Repository Reading Site

tokenizer.json

ml-platform/examples/20-llm-training/03-base-model/tokenizer.json

Text Assetml-platform/examples/20-llm-training/03-base-model/tokenizer.json583 B2026年4月10日 10:41查看原始内容
{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 151643,
      "content": "<|im_start|>",
      "special": true
    },
    {
      "id": 151644,
      "content": "<|im_end|>",
      "special": true
    },
    {
      "id": 151645,
      "content": "<|endoftext|>",
      "special": true
    }
  ],
  "model": {
    "type": "BPE",
    "vocab": {
      "<|im_start|>": 151643,
      "<|im_end|>": 151644,
      "<|endoftext|>": 151645,
      "Kubernetes": 30001,
      "Pod": 30002,
      "LoRA": 30003
    },
    "merges": []
  }
}