Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+epstein_190_24k.jsonl filter=lfs diff=lfs merge=lfs -text
+epstein_190_24k_bio.jsonl filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

biotag.py

@@ -0,0 +1,142 @@
+import json
+import re
+import os
+from typing import List
+from tqdm import tqdm
+
+def normalize(text: str):
+    text = text.lower()
+    text = re.sub(r"[^\w\s]", " ", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+def tokenize_with_spans(text: str):
+    tokens = []
+    spans = []
+
+    for match in re.finditer(r"\w+|[^\w\s]", text):
+        tokens.append(match.group())
+        spans.append((match.start(), match.end()))
+
+    return tokens, spans
+
+def map_norm_to_original(sentence: str):
+    norm = normalize(sentence)
+
+    mapping = []
+    j = 0
+
+    for i, char in enumerate(sentence):
+        if re.match(r"\w", char.lower()):
+            while j < len(norm) and norm[j] == " ":
+                j += 1
+            mapping.append(j)
+            j += 1
+        else:
+            mapping.append(None)
+
+    return mapping, norm
+
+def label_tokens(sentence: str, locations: List[str]):
+    tokens, token_spans = tokenize_with_spans(sentence)
+
+    labels = ["O"] * len(tokens)
+
+    if not locations:
+        return tokens, labels
+
+    mapping, norm_sentence = map_norm_to_original(sentence)
+
+    for loc in locations:
+        norm_loc = normalize(loc)
+
+        # skip empty or garbage locations
+        if not norm_loc or len(norm_loc) < 2:
+            continue
+
+        start = 0
+        while True:
+            idx = norm_sentence.find(norm_loc, start)
+            if idx == -1:
+                break
+
+            norm_start = idx
+            norm_end = idx + len(norm_loc)
+
+            orig_positions = [
+                i for i, m in enumerate(mapping)
+                if m is not None and norm_start <= m < norm_end
+            ]
+
+            if not orig_positions:
+                start = norm_end
+                continue
+
+            start_char = min(orig_positions)
+            end_char = max(orig_positions) + 1
+
+            inside = False
+            for i, (t_start, t_end) in enumerate(token_spans):
+                if t_end <= start_char:
+                    continue
+                if t_start >= end_char:
+                    break
+
+                if not inside:
+                    labels[i] = "B-ADDR"
+                    inside = True
+                else:
+                    labels[i] = "I-ADDR"
+
+            start = norm_end
+
+    return tokens, labels
+
+def count_lines(file_path):
+    with open(file_path, "r", encoding="utf-8") as f:
+        return sum(1 for _ in f)
+
+def clean_filename(path):
+    if not path:
+        return None
+    path = path.replace("\\", "/")
+    return os.path.basename(path)
+
+def process_jsonl(input_path, output_path):
+    total_lines = count_lines(input_path)
+
+    with open(input_path, "r", encoding="utf-8") as infile, \
+         open(output_path, "w", encoding="utf-8") as outfile:
+
+        for line in tqdm(infile, total=total_lines, desc="Processing JSONL"):
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            sentence = entry.get("sentence", "")
+            filename = clean_filename(entry.get("filename"))
+
+            labels_data = entry.get("labels") or {}
+
+            locations = [
+                loc.get("text", "")
+                for loc in labels_data.get("locations", [])
+                if isinstance(loc, dict)
+            ]
+
+            tokens, labels = label_tokens(sentence, locations)
+
+            output_entry = {
+                "filename": filename,
+                "tokens": tokens,
+                "labels": labels
+            }
+
+            outfile.write(json.dumps(output_entry) + "\n")
+
+if __name__ == "__main__":
+    input_file = "/home/laura/rueberta/runs/24k/epstein_190_24k.jsonl"
+    output_file = "/home/laura/rueberta/runs/24k/epstein_190_24k_bio.jsonl"
+
+    process_jsonl(input_file, output_file)

config.json

@@ -0,0 +1,30 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "XLMRobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

epstein_190_24k.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b15c5b27ba587d63bb729bde38bb6820db1917885d9a6f8a3be18a67db924d2a
+size 16953341

epstein_190_24k_bio.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62a7615d1aa40f043ef22dc8d843ff8b5dd009d33bc3563d13bc5305782d4749
+size 27409146

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba1ebd808265070c7759f1d919ddf277ea1e5b9c832374d36090d12bcb7c0bc6
+size 1112269331

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a04dec98c0804beeedde93263778442d924081d88090d4ac3db097205fa959bd
+size 17098351

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}