Test storing internally processed tokens

biolab · Jul 9, 2024 · d9132d6 · d9132d6
1 parent 02f6c3c
commit d9132d6
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/orangecontrib/text/tests/test_bowvectorizer.py b/orangecontrib/text/tests/test_bowvectorizer.py
@@ -31,6 +31,13 @@ def test_empty_tokens(self):
 
         self.assertIs(corpus, bag_of_words)
 
+    def test_store_tokens(self):
+        corpus = Corpus.from_file('deerwester')
+        self.assertFalse(corpus.has_tokens())
+
+        bag_of_words = BowVectorizer().transform(corpus, copy=False)
+        self.assertTrue(bag_of_words.has_tokens())
+
     def test_domain(self):
         vect = BowVectorizer()
         corpus = Corpus.from_file('deerwester')