laurabernardy commited on
Commit
b0b7b9a
·
verified ·
1 Parent(s): f8ddb6c

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
36
  epstein_190_24k.jsonl filter=lfs diff=lfs merge=lfs -text
37
  epstein_190_24k_bio.jsonl filter=lfs diff=lfs merge=lfs -text
38
  tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
36
  epstein_190_24k.jsonl filter=lfs diff=lfs merge=lfs -text
37
  epstein_190_24k_bio.jsonl filter=lfs diff=lfs merge=lfs -text
38
  tokenizer.json filter=lfs diff=lfs merge=lfs -text
39
+ epstein_190_24k_ready.jsonl filter=lfs diff=lfs merge=lfs -text
EStrain.py ADDED
@@ -0,0 +1,376 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import logging
4
+ import torch
5
+ import torch.nn as nn
6
+ from datasets import Dataset
7
+ from transformers import AutoTokenizer, AutoModel
8
+ from torchcrf import CRF
9
+ from torch.utils.data import DataLoader
10
+ from sklearn.metrics import precision_recall_fscore_support
11
+ from huggingface_hub import HfApi, login
12
+ from tqdm import tqdm
13
+
14
+ logging.basicConfig(
15
+ filename="training_rueberta_24k.log",
16
+ filemode="a",
17
+ level=logging.INFO,
18
+ format="%(asctime)s - %(levelname)s - %(message)s"
19
+ )
20
+
21
+ def log(msg):
22
+ print(msg, flush=True)
23
+ logging.info(msg)
24
+
25
+ MODEL_NAME = "xlm-roberta-base"
26
+ BATCH_SIZE = 64
27
+ EPOCHS = 10
28
+ LR = 2e-5
29
+ MAX_LEN = 128
30
+ PATIENCE = 2
31
+ NUM_LOG_SAMPLES = 50
32
+
33
+ OUTPUT_DIR = "/home/laura/rueberta/runs/24k"
34
+ HF_REPO = "laurabernardy/RueBERTa_24k"
35
+
36
+ label_list = ["O", "B-ADDR", "I-ADDR"]
37
+ label2id = {l: i for i, l in enumerate(label_list)}
38
+ id2label = {i: l for l, i in label2id.items()}
39
+
40
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
41
+
42
+ def load_data(path):
43
+ data = []
44
+ with open(path, "r") as f:
45
+ for line in f:
46
+ ex = json.loads(line)
47
+ ex["labels"] = [
48
+ l.replace("GE", "ADDR") if l != "O" else "O"
49
+ for l in ex["labels"]
50
+ ]
51
+ data.append(ex)
52
+ return Dataset.from_list(data)
53
+
54
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
55
+
56
+ def tokenize_and_align(example):
57
+ tokenized = tokenizer(
58
+ example["tokens"],
59
+ is_split_into_words=True,
60
+ truncation=True,
61
+ padding="max_length",
62
+ max_length=MAX_LEN
63
+ )
64
+
65
+ word_ids = tokenized.word_ids()
66
+ labels = []
67
+ prev = None
68
+
69
+ for word_id in word_ids:
70
+ if word_id is None:
71
+ labels.append(-100)
72
+ elif word_id != prev:
73
+ labels.append(label2id[example["labels"][word_id]])
74
+ else:
75
+ label = example["labels"][word_id]
76
+ if label.startswith("B-"):
77
+ label = label.replace("B-", "I-")
78
+ labels.append(label2id[label])
79
+ prev = word_id
80
+
81
+ tokenized["labels"] = labels
82
+ tokenized["word_ids"] = [-1 if word_id is None else word_id for word_id in word_ids]
83
+ return tokenized
84
+
85
+ class XLMRobertaCRF(nn.Module):
86
+ def __init__(self, model_name, num_labels):
87
+ super().__init__()
88
+ self.encoder = AutoModel.from_pretrained(model_name)
89
+ self.dropout = nn.Dropout(0.2)
90
+ self.classifier = nn.Linear(self.encoder.config.hidden_size, num_labels)
91
+ self.crf = CRF(num_labels, batch_first=True)
92
+
93
+ def forward(self, input_ids, attention_mask, labels=None):
94
+ outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
95
+ x = self.dropout(outputs.last_hidden_state)
96
+ emissions = self.classifier(x)
97
+
98
+ mask = attention_mask.bool()
99
+
100
+ if labels is not None:
101
+ labels = labels.clone()
102
+ labels[labels == -100] = 0
103
+ # Normalize by the number of active tokens, producing a familiar
104
+ # per-token loss that is independent of batch and sequence size.
105
+ loss = -self.crf(
106
+ emissions,
107
+ labels,
108
+ mask=mask,
109
+ reduction="token_mean"
110
+ )
111
+ return loss
112
+ else:
113
+ return self.crf.decode(emissions, mask=mask)
114
+
115
+ def save_pretrained(self, save_directory):
116
+ os.makedirs(save_directory, exist_ok=True)
117
+ torch.save(self.state_dict(), os.path.join(save_directory, "pytorch_model.bin"))
118
+ self.encoder.config.save_pretrained(save_directory)
119
+
120
+ def extract_spans(labels):
121
+ spans = []
122
+ start = None
123
+
124
+ for i, label in enumerate(labels):
125
+ if label == "B-ADDR":
126
+ if start is not None:
127
+ spans.append((start, i - 1))
128
+ start = i
129
+ elif label == "I-ADDR" and start is None:
130
+ # Treat an invalid orphan I-tag as the start of an entity.
131
+ start = i
132
+ elif label == "O":
133
+ if start is not None:
134
+ spans.append((start, i - 1))
135
+ start = None
136
+
137
+ if start is not None:
138
+ spans.append((start, len(labels) - 1))
139
+
140
+ return spans
141
+
142
+ def span_overlap(span1, span2):
143
+ s1, e1 = span1
144
+ s2, e2 = span2
145
+ inter = max(0, min(e1, e2) - max(s1, s2) + 1)
146
+ union = max(e1, e2) - min(s1, s2) + 1
147
+ return inter / union if union > 0 else 0
148
+
149
+ def safe_div(numerator, denominator):
150
+ return numerator / denominator if denominator else 0.0
151
+
152
+ def spans_to_character_set(spans, tokens):
153
+ """Return character positions covered by entity spans in ' '.join(tokens)."""
154
+ offsets = []
155
+ cursor = 0
156
+ for token in tokens:
157
+ offsets.append((cursor, cursor + len(token)))
158
+ cursor += len(token) + 1
159
+
160
+ covered = set()
161
+ for start_word, end_word in spans:
162
+ if start_word < len(offsets) and end_word < len(offsets):
163
+ char_start = offsets[start_word][0]
164
+ char_end = offsets[end_word][1]
165
+ covered.update(range(char_start, char_end))
166
+ return covered
167
+
168
+ def main():
169
+ log("Script started")
170
+
171
+ dataset = load_data("/home/laura/rueberta/runs/24k/epstein_190_24k_bio.jsonl")
172
+ dataset = dataset.train_test_split(test_size=0.028)
173
+
174
+ train_dataset = dataset["train"]
175
+ val_dataset = dataset["test"]
176
+
177
+ log("Tokenizing")
178
+ train_dataset = train_dataset.map(tokenize_and_align)
179
+ val_dataset = val_dataset.map(tokenize_and_align)
180
+
181
+ log("Loading model")
182
+ model = XLMRobertaCRF(MODEL_NAME, len(label_list)).to(device)
183
+ log("Model loaded")
184
+
185
+ def collate_fn(batch):
186
+ return {
187
+ "input_ids": torch.tensor([x["input_ids"] for x in batch]).to(device),
188
+ "attention_mask": torch.tensor([x["attention_mask"] for x in batch]).to(device),
189
+ "labels": torch.tensor([x["labels"] for x in batch]).to(device),
190
+ "word_ids": [x["word_ids"] for x in batch],
191
+ "raw_tokens": [x["tokens"] for x in batch],
192
+ }
193
+
194
+ train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, collate_fn=collate_fn)
195
+ val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn)
196
+
197
+ optimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=0.01)
198
+
199
+ def evaluate():
200
+ model.eval()
201
+
202
+ y_true, y_pred = [], []
203
+ exact_tp, exact_gt, exact_pred = 0, 0, 0
204
+ partial_matched_gt = 0
205
+ character_tp, character_gt, character_pred = 0, 0, 0
206
+
207
+ logged = 0
208
+
209
+ with torch.no_grad():
210
+ for batch in val_loader:
211
+ preds = model(batch["input_ids"], batch["attention_mask"])
212
+
213
+ for i in range(len(preds)):
214
+ pred = preds[i]
215
+ true = batch["labels"][i].cpu().tolist()
216
+ word_ids = batch["word_ids"][i]
217
+ raw_tokens = batch["raw_tokens"][i]
218
+
219
+ t_seq, p_seq, tok_seq = [], [], []
220
+ seen_word_ids = set()
221
+
222
+ # Score one label per original word (the first subword).
223
+ for p, t, word_id in zip(pred, true, word_ids):
224
+ if word_id == -1 or word_id in seen_word_ids:
225
+ continue
226
+ seen_word_ids.add(word_id)
227
+
228
+ lt = id2label[t]
229
+ lp = id2label[p]
230
+
231
+ t_seq.append(lt)
232
+ p_seq.append(lp)
233
+ tok_seq.append(raw_tokens[word_id])
234
+
235
+ y_true.append(t)
236
+ y_pred.append(p)
237
+
238
+ gt_spans = extract_spans(t_seq)
239
+ pred_spans = extract_spans(p_seq)
240
+
241
+ gt_span_set = set(gt_spans)
242
+ pred_span_set = set(pred_spans)
243
+ exact_tp += len(gt_span_set & pred_span_set)
244
+ exact_gt += len(gt_span_set)
245
+ exact_pred += len(pred_span_set)
246
+
247
+ partial_matched_gt += sum(
248
+ any(span_overlap(gt, pr) >= 0.5 for pr in pred_spans)
249
+ for gt in gt_spans
250
+ )
251
+
252
+ gt_characters = spans_to_character_set(gt_spans, raw_tokens)
253
+ pred_characters = spans_to_character_set(pred_spans, raw_tokens)
254
+ character_tp += len(gt_characters & pred_characters)
255
+ character_gt += len(gt_characters)
256
+ character_pred += len(pred_characters)
257
+
258
+ if logged < NUM_LOG_SAMPLES:
259
+ log("----- SAMPLE -----")
260
+ log("TOKENS: " + " ".join(tok_seq))
261
+ log("GT: " + " ".join(t_seq))
262
+ log("PRED: " + " ".join(p_seq))
263
+ logged += 1
264
+
265
+ token_precision, token_recall, token_f1, _ = precision_recall_fscore_support(
266
+ y_true,
267
+ y_pred,
268
+ labels=list(range(len(label_list))),
269
+ average="macro",
270
+ zero_division=0
271
+ )
272
+
273
+ exact_precision = safe_div(exact_tp, exact_pred)
274
+ exact_recall = safe_div(exact_tp, exact_gt)
275
+ exact_f1 = safe_div(
276
+ 2 * exact_precision * exact_recall,
277
+ exact_precision + exact_recall
278
+ )
279
+ partial_recall = safe_div(partial_matched_gt, exact_gt)
280
+
281
+ character_precision = safe_div(character_tp, character_pred)
282
+ character_recall = safe_div(character_tp, character_gt)
283
+ character_f1 = safe_div(
284
+ 2 * character_precision * character_recall,
285
+ character_precision + character_recall
286
+ )
287
+
288
+ return {
289
+ "token_precision": token_precision,
290
+ "token_recall": token_recall,
291
+ "token_f1": token_f1,
292
+ "exact_span_precision": exact_precision,
293
+ "exact_span_recall": exact_recall,
294
+ "exact_span_f1": exact_f1,
295
+ "partial_span_recall": partial_recall,
296
+ "character_precision": character_precision,
297
+ "character_recall": character_recall,
298
+ "character_f1": character_f1,
299
+ }
300
+
301
+
302
+ best_score = -1.0
303
+ patience_counter = 0
304
+
305
+ log("Training started")
306
+
307
+ for epoch in range(EPOCHS):
308
+ model.train()
309
+ total_loss = 0
310
+
311
+ for batch in tqdm(train_loader, desc=f"Epoch {epoch+1}", unit="batch"):
312
+ optimizer.zero_grad()
313
+
314
+ loss = model(
315
+ input_ids=batch["input_ids"],
316
+ attention_mask=batch["attention_mask"],
317
+ labels=batch["labels"]
318
+ )
319
+ loss.backward()
320
+
321
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
322
+
323
+ optimizer.step()
324
+ total_loss += loss.item()
325
+
326
+ avg_loss = total_loss / len(train_loader)
327
+ metrics = evaluate()
328
+
329
+ log(
330
+ f"Epoch {epoch+1} | Loss: {avg_loss:.4f} "
331
+ f"| Token Macro Precision: {metrics['token_precision']:.4f} "
332
+ f"| Token Macro Recall: {metrics['token_recall']:.4f} "
333
+ f"| Token Macro F1: {metrics['token_f1']:.4f}\n"
334
+ f"Exact Span Precision: {metrics['exact_span_precision']:.4f} "
335
+ f"| Exact Span Recall: {metrics['exact_span_recall']:.4f} "
336
+ f"| Exact Span F1: {metrics['exact_span_f1']:.4f}\n"
337
+ f"Partial Span Recall (IoU >= 0.5): {metrics['partial_span_recall']:.4f} "
338
+ f"| Character Precision: {metrics['character_precision']:.4f} "
339
+ f"| Character Recall: {metrics['character_recall']:.4f} "
340
+ f"| Character F1: {metrics['character_f1']:.4f}"
341
+ )
342
+
343
+ # Early stopping and checkpoint selection use exact span F1.
344
+ if metrics["exact_span_f1"] > best_score:
345
+ best_score = metrics["exact_span_f1"]
346
+ patience_counter = 0
347
+ log("Saving best model")
348
+ model.save_pretrained(OUTPUT_DIR)
349
+ tokenizer.save_pretrained(OUTPUT_DIR)
350
+ else:
351
+ patience_counter += 1
352
+ log(f"Patience {patience_counter}/{PATIENCE}")
353
+
354
+ if patience_counter >= PATIENCE:
355
+ log("Early stopping")
356
+ break
357
+
358
+ log(f"Finished. Best Exact Span F1: {best_score:.4f}")
359
+
360
+ log("Uploading to Hugging Face")
361
+ login()
362
+
363
+ api = HfApi()
364
+ api.create_repo(repo_id=HF_REPO, exist_ok=True)
365
+
366
+ api.upload_folder(
367
+ folder_path=OUTPUT_DIR,
368
+ repo_id=HF_REPO,
369
+ repo_type="model"
370
+ )
371
+
372
+ log("Upload complete")
373
+
374
+
375
+ if __name__ == "__main__":
376
+ main()
biotag.py CHANGED
@@ -136,7 +136,7 @@ def process_jsonl(input_path, output_path):
136
  outfile.write(json.dumps(output_entry) + "\n")
137
 
138
  if __name__ == "__main__":
139
- input_file = "/home/laura/rueberta/runs/24k/epstein_190_24k.jsonl"
140
  output_file = "/home/laura/rueberta/runs/24k/epstein_190_24k_bio.jsonl"
141
 
142
  process_jsonl(input_file, output_file)
 
136
  outfile.write(json.dumps(output_entry) + "\n")
137
 
138
  if __name__ == "__main__":
139
+ input_file = "/home/laura/rueberta/runs/24k/epstein_190_24k_ready.jsonl"
140
  output_file = "/home/laura/rueberta/runs/24k/epstein_190_24k_bio.jsonl"
141
 
142
  process_jsonl(input_file, output_file)
epstein_190_24k_bio.jsonl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:62a7615d1aa40f043ef22dc8d843ff8b5dd009d33bc3563d13bc5305782d4749
3
- size 27409146
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0b043d2fa0eb37740cc61b0b63c56dc0058e7700664aee1ca1d0592d55743ab
3
+ size 26763675
epstein_190_24k_ready.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e9f1a9bcd00d44d10d9126efe04da4e29f025175520d74d4efe00f7e80e646c
3
+ size 16550178
pytorch_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ba1ebd808265070c7759f1d919ddf277ea1e5b9c832374d36090d12bcb7c0bc6
3
  size 1112269331
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e3cc85e64478cfe960475b00c3a507854b1081eeb14a2550f795c462ae65c10
3
  size 1112269331