Update tokenizer_config.json

#1
Files changed (1) hide show
  1. tokenizer_config.json +5 -5
tokenizer_config.json CHANGED
@@ -59,10 +59,10 @@
59
  "65494": {
60
  "content": "</think>",
61
  "lstrip": false,
62
- "normalized": false,
63
  "rstrip": false,
64
- "single_word": false,
65
- "special": true
66
  },
67
  "65495": {
68
  "content": "source_1",
@@ -397,7 +397,6 @@
397
  "<|im_start|>",
398
  "<|im_end|>",
399
  "<think>",
400
- "</think>",
401
  "source_1",
402
  "source_2",
403
  "source_3",
@@ -443,5 +442,6 @@
443
  "clean_up_tokenization_spaces": true,
444
  "extra_special_tokens": {},
445
  "model_max_length": 1000000000000000019884624838656,
446
- "tokenizer_class": "PreTrainedTokenizer"
 
447
  }
 
59
  "65494": {
60
  "content": "</think>",
61
  "lstrip": false,
62
+ "normalized": true,
63
  "rstrip": false,
64
+ "single_word": true,
65
+ "special": false
66
  },
67
  "65495": {
68
  "content": "source_1",
 
397
  "<|im_start|>",
398
  "<|im_end|>",
399
  "<think>",
 
400
  "source_1",
401
  "source_2",
402
  "source_3",
 
442
  "clean_up_tokenization_spaces": true,
443
  "extra_special_tokens": {},
444
  "model_max_length": 1000000000000000019884624838656,
445
+ "tokenizer_class": "PreTrainedTokenizer",
446
+ "chat_template": "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n<think>\n{% endif %}"
447
  }