Kluge for French stopwords.

I think the problem is with NLTK: it is missing "les" and "a" from its French stopword list.
chnm · Aug 8, 2013 · 6d48f43 · 6d48f43
1 parent 141fe16
commit 6d48f43
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/smartstash/core/utils.py b/smartstash/core/utils.py
@@ -33,7 +33,11 @@
 
 def tokenize(text, lang='en'):
     # if language is not specified or not in our list, fall back to english
-    stopwords = nltk.corpus.stopwords.words(stopword_lang.get(lang, 'english'))
+    stopwords = nltk.corpus.stopwords.words(stopword_lang.get(lang))
+    if lang == 'fr':
+    	stopwords.append('les')
+    	stopwords.append('a')
+
     tokens = nltk.word_tokenize(text)
     words = [w.lower() for w in tokens
              if w.isalnum() and w.lower() not in stopwords]

diff --git a/smartstash/nltk_data/corpora/stopwords/french b/smartstash/nltk_data/corpora/stopwords/french
@@ -15,6 +15,7 @@ il
 je
 la
 le
+les
 leur
 lui
 ma
@@ -116,6 +117,7 @@ eu
 eue
 eues
 eus
+a
 ai
 as
 avons
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,6 +15,7 @@ il @@
     je
     la
     le
+    les
     leur
     lui
     ma
@@ Expand Down Expand Up / @@ -116,6 +117,7 @@ eu @@
     eue
     eues
     eus
+    a
     ai
     as
     avons
@@ Expand Down @@