{
  "model_name": "CodVa",
  "pipeline_version": "codva_it_v4_merged_full",
  "vocab_size": 50269,
  "sequence_length": 2048,
  "eos_token_id": 50256,
  "im_start_id": 50257,
  "im_end_id": 50258,
  "train_samples": 2232347,
  "val_samples": 42496,
  "format": "one_example_per_row_with_loss_mask",
  "label_ignore_index": -100,
  "domain_token_budgets": {
    "code": 600000000,
    "math": 300000000,
    "reasoning": 400000000,
    "general": 300000000
  },
  "domain_token_actuals": {
    "general": 19295136,
    "code": 367961900,
    "math": 211541727,
    "reasoning": 403273403
  },
  "domain_doc_actuals": {
    "general": 123592,
    "code": 641821,
    "math": 626712,
    "reasoning": 732718
  },
  "hours_elapsed": 1.32,
  "collection_stats": {
    "ise-uiuc/Magicoder-OSS-Instruct-75K": {
      "domain": "code",
      "config": null,
      "examples": 65409,
      "skipped": 0,
      "dupes": 9788
    },
    "ise-uiuc/Magicoder-Evol-Instruct-110K": {
      "domain": "code",
      "config": null,
      "examples": 83077,
      "skipped": 0,
      "dupes": 28106
    },
    "m-a-p/CodeFeedback-Filtered-Instruction": {
      "domain": "code",
      "config": null,
      "examples": 87571,
      "skipped": 38,
      "dupes": 68917
    },
    "Vezora/Tested-143k-Python-Alpaca": {
      "domain": "code",
      "config": null,
      "examples": 59623,
      "skipped": 0,
      "dupes": 83704
    },
    "glaiveai/glaive-code-assistant-v3": {
      "domain": "code",
      "config": null,
      "examples": 151040,
      "skipped": 2,
      "dupes": 3993
    },
    "TokenBender/code_instructions_122k_alpaca_style": {
      "domain": "code",
      "config": null,
      "examples": 101376,
      "skipped": 174,
      "dupes": 7
    },
    "sahil2801/CodeAlpaca-20k": {
      "domain": "code",
      "config": null,
      "examples": 15645,
      "skipped": 252,
      "dupes": 4125
    },
    "iamtarun/python_code_instructions_18k_alpaca": {
      "domain": "code",
      "config": null,
      "examples": 3124,
      "skipped": 13,
      "dupes": 15475
    },
    "nickrosh/Evol-Instruct-Code-80k-v1": {
      "domain": "code",
      "config": null,
      "examples": 11164,
      "skipped": 252,
      "dupes": 66848
    },
    "bigcode/self-oss-instruct-sc2-exec-filter-50k": {
      "domain": "code",
      "config": null,
      "examples": 50661,
      "skipped": 0,
      "dupes": 0
    },
    "deepmind/code_contests": {
      "domain": "code",
      "config": null,
      "examples": 13131,
      "skipped": 194,
      "dupes": 3
    },
    "TIGER-Lab/MathInstruct": {
      "domain": "math",
      "config": null,
      "examples": 151040,
      "skipped": 1,
      "dupes": 8783
    },
    "gsm8k": {
      "domain": "math",
      "config": "main",
      "examples": 7256,
      "skipped": 0,
      "dupes": 217
    },
    "microsoft/orca-math-word-problems-200k": {
      "domain": "math",
      "config": null,
      "examples": 150016,
      "skipped": 2,
      "dupes": 5453
    },
    "math_qa": {
      "domain": "math",
      "config": null,
      "examples": 0,
      "skipped": 0,
      "dupes": 0
    },
    "ChilleD/SVAMP": {
      "domain": "math",
      "config": null,
      "examples": 700,
      "skipped": 0,
      "dupes": 0
    },
    "garage-bAInd/Open-Platypus": {
      "domain": "math",
      "config": null,
      "examples": 16644,
      "skipped": 5724,
      "dupes": 2558
    },
    "AI-MO/NuminaMath-CoT": {
      "domain": "math",
      "config": null,
      "examples": 100352,
      "skipped": 0,
      "dupes": 12026
    },
    "meta-math/MetaMathQA": {
      "domain": "math",
      "config": null,
      "examples": 200704,
      "skipped": 0,
      "dupes": 9554
    },
    "Open-Orca/SlimOrca": {
      "domain": "reasoning",
      "config": null,
      "examples": 151040,
      "skipped": 0,
      "dupes": 475
    },
    "teknium/OpenHermes-2.5": {
      "domain": "reasoning",
      "config": null,
      "examples": 200704,
      "skipped": 0,
      "dupes": 16716
    },
    "Open-Orca/OpenOrca": {
      "domain": "reasoning",
      "config": null,
      "examples": 151040,
      "skipped": 0,
      "dupes": 16972
    },
    "LDJnr/Capybara": {
      "domain": "reasoning",
      "config": null,
      "examples": 14133,
      "skipped": 0,
      "dupes": 1873
    },
    "HuggingFaceH4/no_robots": {
      "domain": "general",
      "config": null,
      "examples": 9500,
      "skipped": 0,
      "dupes": 0
    },
    "timdettmers/openassistant-guanaco": {
      "domain": "reasoning",
      "config": null,
      "examples": 9825,
      "skipped": 0,
      "dupes": 1
    },
    "openbmb/UltraInteract_sft": {
      "domain": "reasoning",
      "config": null,
      "examples": 84480,
      "skipped": 0,
      "dupes": 16416
    },
    "WizardLM/WizardLM_evol_instruct_V2_196k": {
      "domain": "reasoning",
      "config": null,
      "examples": 73368,
      "skipped": 1,
      "dupes": 16400
    },
    "Anthropic/hh-rlhf": {
      "domain": "general",
      "config": "harmless-base",
      "examples": 0,
      "skipped": 0,
      "dupes": 0
    },
    "HuggingFaceH4/ultrachat_200k": {
      "domain": "reasoning",
      "config": null,
      "examples": 48128,
      "skipped": 0,
      "dupes": 0
    },
    "databricks/databricks-dolly-15k": {
      "domain": "general",
      "config": null,
      "examples": 14862,
      "skipped": 129,
      "dupes": 20
    },
    "yahma/alpaca-cleaned": {
      "domain": "general",
      "config": null,
      "examples": 51200,
      "skipped": 130,
      "dupes": 3
    }
  },
  "special_tokens": {
    "<|im_start|>": 50257,
    "<|im_end|>": 50258,
    "<|code_start|>": 50259,
    "<|code_end|>": 50260,
    "<|math_start|>": 50261,
    "<|math_end|>": 50262,
    "<|solution|>": 50263,
    "<|doc_end|>": 50264,
    "<|domain_code|>": 50265,
    "<|domain_math|>": 50266,
    "<|domain_general|>": 50267,
    "<|domain_reasoning|>": 50268,
    "<|endoftext|>": 50256
  },
  "anchor_repeat": 3,
  "note": "v2 + v3 datasets fully merged. Overlap with pretrain is intentional \u2014 SFT teaches format/turn-taking on known content. Identity anchors repeated 3x to ensure CodVa name/persona locks in. Train with 2-3 epochs for best results on this dataset size. | UltraChat 200k train_sft appended: 150,000 examples (was 2,082,347 \u2192 now 2,232,347).",
  "ultrachat_appended": true,
  "ultrachat_examples": 150000,
  "ultrachat_skipped": 0,
  "ultrachat_dupes": 0
}