Rocketknight1 HF Staff commited on
Commit
277d410
·
verified ·
1 Parent(s): 6e8c319

Add response_template to tokenizer_config.json

Browse files

Updates the `response_template` key in `tokenizer_config.json`.

Files changed (1) hide show
  1. tokenizer_config.json +46 -1
tokenizer_config.json CHANGED
@@ -63,7 +63,52 @@
63
  "type": "object",
64
  "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
65
  },
66
- "response_template": {"defaults": {"role": "assistant"}, "start_anchor": ["<|turn>model\n", "<tool_response|>"], "fields": {"thinking": {"open": "<|channel>thought\n", "close": "<channel|>", "content": "text"}, "tool_calls": {"open_pattern": "<\\|tool_call>call:(?P<name>\\w+)", "close": "<tool_call|>", "repeats": true, "content": "json", "content_args": {"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}, "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}}, "content": {"close": ["<turn|>", "<|tool_response>", "<eos>"], "content": "text"}}},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  "soc_token": "<|channel>",
68
  "sot_token": "<|turn>",
69
  "stc_token": "<|tool_call>",
 
63
  "type": "object",
64
  "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
65
  },
66
+ "response_template": {
67
+ "defaults": {
68
+ "role": "assistant"
69
+ },
70
+ "fields": {
71
+ "content": {
72
+ "close": [
73
+ "<turn|>",
74
+ "<|tool_response>",
75
+ "<eos>"
76
+ ],
77
+ "content": "text"
78
+ },
79
+ "thinking": {
80
+ "close": "<channel|>",
81
+ "content": "text",
82
+ "open": "<|channel>thought\n"
83
+ },
84
+ "tool_calls": {
85
+ "close": "<tool_call|>",
86
+ "content": "json",
87
+ "content_args": {
88
+ "string_delims": [
89
+ [
90
+ "<|\"|>",
91
+ "<|\"|>"
92
+ ]
93
+ ],
94
+ "unquoted_keys": true
95
+ },
96
+ "open_pattern": "<\\|tool_call>call:(?P<name>\\w+)",
97
+ "repeats": true,
98
+ "transform": {
99
+ "function": {
100
+ "arguments": "{content}",
101
+ "name": "{name}"
102
+ },
103
+ "type": "function"
104
+ }
105
+ }
106
+ },
107
+ "start_anchor": [
108
+ "<|turn>model\n",
109
+ "<tool_response|>"
110
+ ]
111
+ },
112
  "soc_token": "<|channel>",
113
  "sot_token": "<|turn>",
114
  "stc_token": "<|tool_call>",