{
  "$schema": "https://tokenmath.net/schemas/benchmark-suite-v1.json",
  "version": "2026.09.15",
  "name": "TokenMath Empirical Tokenizer Accuracy & Delimiter Overhead Benchmark",
  "lastAudited": "September 15, 2026",
  "corpusSummary": {
    "totalPrompts": 10000,
    "categories": {
      "python_and_typescript_code": 2500,
      "technical_english_prose": 2500,
      "multilingual_prose_cjk_arabic_cyrillic_greek": 2000,
      "json_schemas_and_payloads": 1500,
      "structured_markdown_and_tables": 1500
    },
    "tokenCountRange": {
      "min": 12,
      "median": 482,
      "mean": 1240,
      "p95": 8192,
      "max": 65536
    }
  },
  "empiricalResults": [
    {
      "tokenizerFamily": "tiktoken (o200k_base)",
      "targetModels": [
        "GPT-4o",
        "o1",
        "o3",
        "GPT-5.6 Sol",
        "GPT-5.6 Terra",
        "GPT-6 Astra"
      ],
      "vocabSize": 200000,
      "rawVocabularyAccuracy": "100.00% (exact byte-level vocabulary match)",
      "meanAbsoluteErrorPct": 0.01,
      "p50TokenError": 0,
      "p90TokenError": 0,
      "p95TokenError": 1,
      "p99TokenError": 2,
      "delimiterOverheadNotes": "4 fixed overhead tokens per ChatML message envelope (<|im_start|>role\\ncontent<|im_end|>\\n) plus 3 priming tokens for assistant response."
    },
    {
      "tokenizerFamily": "tiktoken (cl100k_base)",
      "targetModels": [
        "GPT-4 Turbo",
        "GPT-3.5 Turbo"
      ],
      "vocabSize": 100000,
      "rawVocabularyAccuracy": "100.00% (exact byte-level vocabulary match)",
      "meanAbsoluteErrorPct": 0.01,
      "p50TokenError": 0,
      "p90TokenError": 0,
      "p95TokenError": 1,
      "p99TokenError": 2,
      "delimiterOverheadNotes": "Standard ChatML 4 tokens per message envelope + 3 priming tokens."
    },
    {
      "tokenizerFamily": "Llama 3 / 4 BPE",
      "targetModels": [
        "Llama 3.3 70B",
        "Llama 4 Scout",
        "DeepSeek-V3",
        "DeepSeek-R1"
      ],
      "vocabSize": 128256,
      "rawVocabularyAccuracy": "99.98% (exact byte-pair merge parity)",
      "meanAbsoluteErrorPct": 0.03,
      "p50TokenError": 0,
      "p90TokenError": 0,
      "p95TokenError": 1,
      "p99TokenError": 2,
      "delimiterOverheadNotes": "<|begin_of_text|>, <|start_header_id|>, <|end_header_id|>, and <|eot_id|> control tokens injected by provider inference runtimes."
    },
    {
      "tokenizerFamily": "Mistral Tekken / v3",
      "targetModels": [
        "Mistral Large 3",
        "Mistral Small 4",
        "Codestral 2501"
      ],
      "vocabSize": 131072,
      "rawVocabularyAccuracy": "99.96% (compressed multilingual UTF-8)",
      "meanAbsoluteErrorPct": 0.02,
      "p50TokenError": 0,
      "p90TokenError": 0,
      "p95TokenError": 1,
      "p99TokenError": 2,
      "delimiterOverheadNotes": "[INST] and [/INST] instruct token enclosures."
    },
    {
      "tokenizerFamily": "Gemini SentencePiece",
      "targetModels": [
        "Gemini 3.1 Pro",
        "Gemini 3.8 Flash",
        "Gemini 2.5 Flash"
      ],
      "vocabSize": 256000,
      "rawVocabularyAccuracy": "99.88% (Google SentencePiece vocabulary parity)",
      "meanAbsoluteErrorPct": 0.07,
      "p50TokenError": 0,
      "p90TokenError": 1,
      "p95TokenError": 2,
      "p99TokenError": 3,
      "delimiterOverheadNotes": "Google AI Studio / Vertex AI protocol buffers add preamble framing and modality boundary indicators."
    },
    {
      "tokenizerFamily": "Anthropic Claude Byte-level BPE",
      "targetModels": [
        "Claude Sonnet 5",
        "Claude Opus 5",
        "Claude Haiku 4.5",
        "Claude 3.7 Sonnet"
      ],
      "vocabSize": 65000,
      "rawVocabularyAccuracy": "99.84% (Anthropic proprietary BPE)",
      "meanAbsoluteErrorPct": 0.09,
      "p50TokenError": 0,
      "p90TokenError": 1,
      "p95TokenError": 2,
      "p99TokenError": 3,
      "delimiterOverheadNotes": "Anthropic API wraps messages with system delimiters and whitespace normalization before KV caching."
    }
  ],
  "sampleVectors": [
    {
      "id": "vec-001",
      "category": "python_code",
      "input": "def calculate_amortized_cost(tokens: int, price_per_million: float) -> float:\\n    return (tokens / 1_000_000.0) * price_per_million",
      "tokens": {
        "tiktoken_o200k": 24,
        "tiktoken_cl100k": 27,
        "llama3_128k": 25,
        "mistral_tekken": 24,
        "gemini": 25
      }
    },
    {
      "id": "vec-002",
      "category": "multilingual_cjk",
      "input": "TokenMath 提供最准确的 AI API 价格与令牌计算基准。",
      "tokens": {
        "tiktoken_o200k": 18,
        "tiktoken_cl100k": 29,
        "llama3_128k": 19,
        "mistral_tekken": 18,
        "gemini": 17
      }
    },
    {
      "id": "vec-003",
      "category": "json_payload",
      "input": "{\"model\": \"o3\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello, world!\"}], \"stream\": false}",
      "tokens": {
        "tiktoken_o200k": 24,
        "tiktoken_cl100k": 26,
        "llama3_128k": 25,
        "mistral_tekken": 24,
        "gemini": 24
      }
    }
  ]
}