Hugging Face's logo

PleIAs
/
Baguettotron

Text Generation
Transformers
Safetensors
llama
conversational
text-generation-inference
Model card Files
xet
 Community
9
pdelobelle commited on
Commit
504b4c1
·
1 Parent(s): 00055f2

Upload 3 files (#8)

Browse files

- Upload 3 files (dc52edbf200209ee995d7ff89ba483cd3b605693)

Files changed (3) hide show
  1. special_tokens_map.json +2 -7
  2. tokenizer.json +3 -3
  3. tokenizer_config.json +6 -10
special_tokens_map.json CHANGED
@@ -1,9 +1,8 @@
1
  {
2
  "additional_special_tokens": [
3
  "<|im_start|>",
4
- "<|im_end>",
5
  "<think>",
6
- "</think>",
7
  "source_1",
8
  "source_2",
9
  "source_3",
@@ -45,9 +44,5 @@
45
  "⟨H≈1.6⟩",
46
  "⟨H≈1.7⟩",
47
  "⟨H≈1.8⟩"
48
- ],
49
- "bos_token": "<|begin_of_text|>",
50
- "eos_token": "<|end_of_text|>",
51
- "pad_token": "[PAD]",
52
- "unk_token": "[UNK]"
53
  }
 
1
  {
2
  "additional_special_tokens": [
3
  "<|im_start|>",
4
+ "<|im_end|>",
5
  "<think>",
 
6
  "source_1",
7
  "source_2",
8
  "source_3",
 
44
  "⟨H≈1.6⟩",
45
  "⟨H≈1.7⟩",
46
  "⟨H≈1.8⟩"
47
+ ]
 
 
 
 
48
  }
tokenizer.json CHANGED
@@ -50,7 +50,7 @@
50
  },
51
  {
52
  "id": 65492,
53
- "content": "<|im_end>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
@@ -69,11 +69,11 @@
69
  {
70
  "id": 65494,
71
  "content": "</think>",
72
- "single_word": false,
73
  "lstrip": false,
74
  "rstrip": false,
75
  "normalized": false,
76
- "special": true
77
  },
78
  {
79
  "id": 65495,
 
50
  },
51
  {
52
  "id": 65492,
53
+ "content": "<|im_end|>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
 
69
  {
70
  "id": 65494,
71
  "content": "</think>",
72
+ "single_word": true,
73
  "lstrip": false,
74
  "rstrip": false,
75
  "normalized": false,
76
+ "special": false
77
  },
78
  {
79
  "id": 65495,
tokenizer_config.json CHANGED
@@ -41,7 +41,7 @@
41
  "special": true
42
  },
43
  "65492": {
44
- "content": "<|im_end>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
@@ -59,10 +59,10 @@
59
  "65494": {
60
  "content": "</think>",
61
  "lstrip": false,
62
- "normalized": false,
63
  "rstrip": false,
64
- "single_word": false,
65
- "special": true
66
  },
67
  "65495": {
68
  "content": "source_1",
@@ -395,9 +395,8 @@
395
  },
396
  "additional_special_tokens": [
397
  "<|im_start|>",
398
- "<|im_end>",
399
  "<think>",
400
- "</think>",
401
  "source_1",
402
  "source_2",
403
  "source_3",
@@ -440,12 +439,9 @@
440
  "⟨H≈1.7⟩",
441
  "⟨H≈1.8⟩"
442
  ],
443
- "bos_token": "<|begin_of_text|>",
444
  "clean_up_tokenization_spaces": true,
445
- "eos_token": "<|end_of_text|>",
446
  "extra_special_tokens": {},
447
  "model_max_length": 1000000000000000019884624838656,
448
- "pad_token": "[PAD]",
449
  "tokenizer_class": "PreTrainedTokenizer",
450
- "unk_token": "[UNK]"
451
  }
 
41
  "special": true
42
  },
43
  "65492": {
44
+ "content": "<|im_end|>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
 
59
  "65494": {
60
  "content": "</think>",
61
  "lstrip": false,
62
+ "normalized": true,
63
  "rstrip": false,
64
+ "single_word": true,
65
+ "special": false
66
  },
67
  "65495": {
68
  "content": "source_1",
 
395
  },
396
  "additional_special_tokens": [
397
  "<|im_start|>",
398
+ "<|im_end|>",
399
  "<think>",
 
400
  "source_1",
401
  "source_2",
402
  "source_3",
 
439
  "⟨H≈1.7⟩",
440
  "⟨H≈1.8⟩"
441
  ],
 
442
  "clean_up_tokenization_spaces": true,
 
443
  "extra_special_tokens": {},
444
  "model_max_length": 1000000000000000019884624838656,
 
445
  "tokenizer_class": "PreTrainedTokenizer",
446
+ "chat_template": "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n<think>\n{% endif %}"
447
  }