Upload folder using huggingface_hub

README.md

@@ -1793,9 +1793,9 @@ def get_new_queries(queries, query_max_len, examples_prefix, tokenizer):
         return_tensors=None,
         add_special_tokens=False
     )
-    prefix_ids = tokenizer(examples_prefix)['input_ids']
-    suffix_ids = tokenizer('\n<response>')['input_ids']
-    new_max_length = (len(prefix_ids) + len(suffix_ids) + query_max_len) // 8 * 8 + 8
+    prefix_ids = tokenizer(examples_prefix, add_special_tokens=False)['input_ids']
+    suffix_ids = tokenizer('\n<response>', add_special_tokens=False)['input_ids']
+    new_max_length = (len(prefix_ids) + len(suffix_ids) + query_max_len + 8) // 8 * 8 + 8
     new_queries = tokenizer.batch_decode(inputs['input_ids'])
     for i in range(len(new_queries)):
         new_queries[i] = examples_prefix + new_queries[i] + '\n<response>'