-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathembedding_fasttext.py
191 lines (137 loc) · 6.46 KB
/
embedding_fasttext.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
"""
embedding method: FastText by Facebook
- using subword information: character n-grams
wrapper: https://github.com/salestock/fastText.py (pip install fasttext)
https://pypi.python.org/pypi/fasttext
"""
import fasttext
import numpy as np
import os
import json
from embedding_base import EmbeddingAbstractBase
from gensim.models import KeyedVectors
import multiprocessing
class EmbeddingFasttext(EmbeddingAbstractBase):
def __init__(self, config):
self._model = None
self._vectors = None
self.config = config
def train_model(self, train_data_src, emb_model_dir, emb_model_fn):
"""
Train fasttext model with following parameters:
***********************************************
input_file training file path (required)
output output file path (required)
lr learning rate [0.05]
lr_update_rate change the rate of updates for the learning rate [100]
dim size of word vectors [100]
ws size of the context window [5]
epoch number of epochs [5]
min_count minimal number of word occurences [5]
neg number of negatives sampled [5]
word_ngrams max length of word ngram [1]
loss loss function {ns, hs, softmax} [ns]
bucket number of buckets [2000000]
minn min length of char ngram [3]
maxn max length of char ngram [6]
thread number of threads [12]
t sampling threshold [0.0001]
silent disable the log output from the C++ extension [1]
encoding specify input_file encoding [utf-8]
"""
# remove extention for fasttext
model_name, ext = os.path.splitext(emb_model_fn)
emb_model_src_no_ext = os.path.join(emb_model_dir, model_name)
# Model parameters
algorithm = self.config.config['embedding_algorithm'] # skipgram or cbow
print("Embedding Method: fastText, Algorithm:", algorithm)
# embedding vector dimension
emb_dim = self.config.config['embedding_vector_dim']
# minimum number a token has to appear to be included in _model
min_count = self.config.config['min_token_appearance']
# min ngram length (number of chars)
minn = 3
# max ngram length (number of chars)
maxn = 6
# number of cores
n_cores = multiprocessing.cpu_count()
# train model
if algorithm == "skipgram":
self._model = fasttext.skipgram(input_file = train_data_src,
output = emb_model_src_no_ext,
dim = emb_dim,
min_count = min_count,
minn = minn,
maxn = maxn,
thread = n_cores,
silent = 0)
self._vectors = KeyedVectors.load_word2vec_format(emb_model_src_no_ext + '.vec')
elif algorithm == "cbow":
self._model = fasttext.cbow(input_file = train_data_src,
output = emb_model_src_no_ext,
dim = emb_dim,
min_count = min_count,
minn = minn,
maxn = maxn,
thread = n_cores,
silent = 0)
self._vectors = KeyedVectors.load_word2vec_format(emb_model_src_no_ext + '.vec')
else:
print("fasttext algorithm must be 'skipgram' or 'cbow' ")
return AttributeError
# save configuration
config_fn = model_name + '_configuration.json'
config_src = os.path.join(emb_model_dir, config_fn)
with open(config_src, 'w') as f:
json.dump(self.config.config, f, indent=4)
# get total number of words in trainfile
n_lines = 0
n_words = 0
with open(train_data_src) as f:
for line in f.readlines():
n_lines += 1
n_words += len(line.split())
# note that word2vec counts newline chars as words, we do the same here for consistency
# to compare these two word counts
print("Training finsihed.\n"
"Vocab size:", len(self.get_vocab()), ", words in train file:", n_lines + n_words, "\n"
"Model saved at:", emb_model_src_no_ext + ".bin,", emb_model_src_no_ext + ".vec" )
def similarity(self, word1, word2):
if not self._model:
print("Model not defined. Train or load a model.")
return ReferenceError
return self._model.cosine_similarity(word1, word2)
def most_similar_n(self, word, topn=10):
if not self._model:
print("Model not defined. Train or load a model.")
return ReferenceError
word_vec = self.word_vec(word)
return self._vectors.similar_by_vector(word_vec, topn)
def load_model(self, emb_model_dir, emb_model_fn):
model_name, ext = os.path.splitext(emb_model_fn)
emb_model_src_no_ext = os.path.join(emb_model_dir, model_name)
self._model = fasttext.load_model(emb_model_src_no_ext + '.bin')
self._vectors = KeyedVectors.load_word2vec_format(emb_model_src_no_ext + '.vec')
def get_vocab(self):
if not self._model:
print("Model not defined. Train or load a model.")
return ReferenceError
return list(self._model.words)
def word_vec(self, word):
if not self._model:
print("Model not defined. Train or load a model.")
return ReferenceError
return np.asarray(self._model[word], dtype=np.float64)
def vec_dim(self):
if not self._model:
print("Model not defined. Train or load a model.")
return ReferenceError
return self._model.dim
def analogy(self, positives, negatives, topn):
if not self._model:
print("Model not defined. Train or load a model.")
return ReferenceError
return self._vectors.most_similar(positives, negatives, topn)
def may_construct_word_vec(self, word):
# TODO implement, assumes that fasttext can construct all words at the moment
return True