moved tokenize.py file to fix installation issues (via pip from github)

rouge/README.md

@@ -1,5 +1,26 @@
 # Python ROUGE Implementation
 
+## Installation
+
+```
+pip install -e "git+https://github.com/mim-solutions/google-research.git#egg=rouge_score&subdirectory=rouge
+```
+
+## Fork
+
+The original repo (https://github.com/google-research/google-research) was forked
+due to problems with installing it:
+
+* version available in pip (https://pypi.org/project/rouge-score/) is outdated,
+* version from master failed to install: it had file `tokenize.py` and it hide standard library module `tokenize`, 
+so installation scripts failed with error like: 
+
+```
+File "/home/jupyter-apacuk/.conda/envs/encoder_decoder/lib/python3.8/linecache.py", line 136, in updatecache
+    with tokenize.open(fullname) as fp:
+AttributeError: module 'tokenize' has no attribute 'open'
+```
+
 ## Overview
 
 This is a native python implementation of ROUGE, designed to replicate results

rouge/io_test.py

@@ -22,10 +22,10 @@
 import tempfile
 
 from absl.testing import absltest
-from rouge import io
-from rouge import rouge_scorer
-from rouge import scoring
-from rouge import test_util
+from rouge_score import io
+from rouge_score import rouge_scorer
+from rouge_score import scoring
+from rouge_score import test_util
 
 
 class IoTest(absltest.TestCase):

rouge/rouge.py

@@ -41,9 +41,9 @@
 
 from absl import app
 from absl import flags
-from rouge import io
-from rouge import rouge_scorer
-from rouge import scoring
+from rouge_score import io
+from rouge_score import rouge_scorer
+from rouge_score import scoring
 
 flags.DEFINE_string("target_filepattern", None,
                     "Files containing target text.")

rouge/rouge_scorer.py

@@ -43,8 +43,8 @@
 import six
 from six.moves import map
 from six.moves import range
-from rouge import scoring
-from rouge import tokenizers
+from rouge_score import scoring
+from rouge_score import tokenizers
 
 
 class RougeScorer(scoring.BaseScorer):

rouge/rouge_scorer_test.py

@@ -29,9 +29,9 @@
 
 from absl.testing import absltest
 from absl.testing import parameterized
-from rouge import rouge_scorer
-from rouge import test_util
-from rouge import tokenizers
+from rouge_score import rouge_scorer
+from rouge_score import test_util
+from rouge_score import tokenizers
 
 
 class RougeScorerTest(parameterized.TestCase):

rouge/run.sh

@@ -20,6 +20,11 @@ virtualenv -p python3 .
 source ./bin/activate
 
 pip install -r rouge/requirements.txt
+
+
+python -c "import nltk; nltk.download('punkt')"
 python -m rouge.io_test
 python -m rouge.rouge_scorer_test
 python -m rouge.scoring_test
+python -m rouge.tokenize_lib_test
+python -m rouge.tokenizers_test

rouge/scoring_test.py

@@ -29,9 +29,9 @@
 import numpy as np
 from six.moves import range
 from six.moves import zip
-from rouge import rouge_scorer
-from rouge import scoring
-from rouge import test_util
+from rouge_score import rouge_scorer
+from rouge_score import scoring
+from rouge_score import test_util
 
 # Delta for matching against ground truth rouge values. Must be relatively
 # high compared to the individual rouge tests since bootstrap sampling

rouge/setup.py

@@ -20,7 +20,7 @@
 
 setuptools.setup(
     name="rouge_score",
-    version="0.0.4",
+    version="0.0.5",
     author="Google LLC",
     author_email="[email protected]",
     description="Pure python implementation of ROUGE-1.5.5.",

rouge/tokenize_test.py → rouge/tokenize_lib_test.py

@@ -13,23 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Tests for tokenize."""
+"""Tests for tokenize_lib."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
 from absl.testing import absltest
-from rouge import tokenize
+from rouge_score import tokenize_lib
 
 
 class TokenizeTest(absltest.TestCase):
 
   def test_give_me_a_name(self):
     self.assertEqual(['one', 'two', 'three'],
-                     tokenize.tokenize('one Two three', None))
+                     tokenize_lib.tokenize('one Two three', None))
     self.assertEqual(['one', 'two', 'three'],
-                     tokenize.tokenize('one\n Two \nthree', None))
+                     tokenize_lib.tokenize('one\n Two \nthree', None))
 
 
 if __name__ == '__main__':

rouge/tokenizers.py

@@ -21,7 +21,7 @@
 """
 import abc
 from nltk.stem import porter
-from rouge import tokenize
+from rouge_score import tokenize_lib
 
 
 class Tokenizer(abc.ABC):
@@ -48,4 +48,4 @@ def __init__(self, use_stemmer=False):
     self._stemmer = porter.PorterStemmer() if use_stemmer else None
 
   def tokenize(self, text):
-    return tokenize.tokenize(text, self._stemmer)
+    return tokenize_lib.tokenize(text, self._stemmer)

rouge/tokenizers_test.py

@@ -16,7 +16,7 @@
 """Tests for tokenizers."""
 
 from absl.testing import absltest
-from rouge import tokenizers
+from rouge_score import tokenizers
 
 
 class TokenizersTest(absltest.TestCase):