Correct vector-table provenance in meta.json (floret-torch, fawiki corpus)
Browse files
meta.json
CHANGED
|
@@ -1,33 +1,60 @@
|
|
| 1 |
{
|
| 2 |
-
"lang":
|
| 3 |
-
"name":
|
| 4 |
-
"version":
|
| 5 |
-
"description":
|
| 6 |
-
"author":
|
| 7 |
-
"email":
|
| 8 |
-
"url":
|
| 9 |
-
"license":
|
| 10 |
-
"sources":
|
| 11 |
{
|
| 12 |
-
"name":
|
| 13 |
-
"url":
|
| 14 |
-
"author":
|
| 15 |
-
"license":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
}
|
| 17 |
],
|
| 18 |
-
"spacy_version":
|
| 19 |
-
"spacy_git_version":
|
| 20 |
-
"vectors":
|
| 21 |
-
"width":
|
| 22 |
-
"vectors":
|
| 23 |
-
"keys":
|
| 24 |
-
"name":
|
|
|
|
|
|
|
|
|
|
| 25 |
},
|
| 26 |
-
"
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
"
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
"spacy>=3.8.15,<3.9.0"
|
| 32 |
-
]
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
{
|
| 2 |
+
"lang":"fa",
|
| 3 |
+
"name":"floret_400k",
|
| 4 |
+
"version":"0.1.0",
|
| 5 |
+
"description":"Persian floret static vector table: 50,000 rows x 300 dimensions, floret mode, minn=maxn=5, hash_count=2, trained with floret-torch on the first 400,000 articles of the Persian Wikipedia dump. Vectors only, no pipeline components. This is the table used by the fa_*_news_md tier.",
|
| 6 |
+
"author":"Kiyarash Fazeli",
|
| 7 |
+
"email":"kiyarash@nlogn.ir",
|
| 8 |
+
"url":"https://github.com/Fazel94/spacy-persian",
|
| 9 |
+
"license":"CC BY-SA 4.0",
|
| 10 |
+
"sources":[
|
| 11 |
{
|
| 12 |
+
"name":"fa_floret static vectors (50k rows x 300d, floret mode, first 400,000 Persian Wikipedia articles, trained with floret-torch)",
|
| 13 |
+
"url":"https://github.com/Fazel94/spacy-persian",
|
| 14 |
+
"author":"Kiyarash Fazeli",
|
| 15 |
+
"license":"CC BY-SA 4.0"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"name":"Persian Wikipedia dump (fawiki), the text the vector table is trained on",
|
| 19 |
+
"url":"https://dumps.wikimedia.org/fawiki/",
|
| 20 |
+
"author":"Wikipedia contributors",
|
| 21 |
+
"license":"CC BY-SA 4.0"
|
| 22 |
}
|
| 23 |
],
|
| 24 |
+
"spacy_version":">=3.8.15,<3.9.0",
|
| 25 |
+
"spacy_git_version":"f69c32f",
|
| 26 |
+
"vectors":{
|
| 27 |
+
"width":300,
|
| 28 |
+
"vectors":50000,
|
| 29 |
+
"keys":-1,
|
| 30 |
+
"name":"floret"
|
| 31 |
+
},
|
| 32 |
+
"labels":{
|
| 33 |
+
|
| 34 |
},
|
| 35 |
+
"pipeline":[
|
| 36 |
+
|
| 37 |
+
],
|
| 38 |
+
"components":[
|
| 39 |
+
|
| 40 |
+
],
|
| 41 |
+
"disabled":[
|
| 42 |
+
|
| 43 |
+
],
|
| 44 |
+
"requirements":[
|
| 45 |
+
"spacy>=3.8.15,<3.9.0",
|
| 46 |
"spacy>=3.8.15,<3.9.0"
|
| 47 |
+
],
|
| 48 |
+
"vectors_training":{
|
| 49 |
+
"corpus":"first 400,000 articles of the Persian Wikipedia (`fawiki`) dump, extracted with WikiExtractor `--no-templates`, sentence-tokenized with spaCy `blank(\"fa\")`",
|
| 50 |
+
"tokens":"~163M (project estimate for 400,000 articles; not measured for this run)",
|
| 51 |
+
"epochs":5,
|
| 52 |
+
"minn":5,
|
| 53 |
+
"maxn":5,
|
| 54 |
+
"hash_count":2,
|
| 55 |
+
"min_count":20,
|
| 56 |
+
"tool":"[floret-torch](https://github.com/Fazel94/floret-torch) (GPU port of [explosion/floret](https://github.com/explosion/floret)), cbow, on a Colab T4",
|
| 57 |
+
"cli":"python -m floret_torch.train --input fa.txt --output fa \\\n --model cbow --mode floret --dim 300 --minn 5 --maxn 5 \\\n --neg 10 --epoch 5 --hashCount 2 --bucket 50000 \\\n --minCount 20 --lr 0.05 --batch 8192 --dtype fp32 --device cuda --seed 0",
|
| 58 |
+
"dump_date":"not pinned: the run used `wikipedia_date: latest`"
|
| 59 |
+
}
|
| 60 |
+
}
|