Travis update (#292)

* Update Travis CI

* Update versions of pylint, flake8, and mypy

* Fix mypy error

* Fix pylint error

.pylintrc

@@ -79,6 +79,7 @@ disable=invalid-name,too-many-branches,too-many-statements,too-many-arguments,
         no-self-use,                                                            # (cannot ignore overridden methods)
         unused-wildcard-import,                                                 # (https://github.com/rogalski/astroid/commit/82c6ef644a2efb77217a23d9b8a6cfb5caffb4ba)
         duplicate-code,                                                         # (will be fixed in next release)
+        import-outside-toplevel,
 
 
 [REPORTS]

.travis.yml

@@ -1,4 +1,4 @@
-sudo: required
+os: linux
 language: python
 dist: xenial  # Required for Python 3.7
 cache: pip
@@ -14,27 +14,15 @@ env:
   - TORCH_VER="1.3.0"
   - TORCH_VER="1.4.0"
 
-matrix:
-  fast_finish: true
-  exclude:
-    - python: "3.6"
-      env: TORCH_VER="1.0.1"
-    - python: "3.6"
-      env: TORCH_VER="1.2.0"
-    - python: "3.6"
-      env: TORCH_VER="1.3.0"
-    - python: "3.6"
-      env: TORCH_VER="1.4.0"
-
 install:
   - pip install --upgrade pip
   - pip install --progress-bar off torch==$TORCH_VER
   - pip install --progress-bar off .[extras]
   - if [[ $TORCH_VER == "1.4.0" ]]; then
-      pip install pylint==2.3.1 flake8==3.7.7;
+      pip install pylint==2.4.4 flake8==3.7.9;
     fi
   - if [[ $TORCH_VER != "1.0.1" ]]; then
-      pip install mypy==0.720;
+      pip install mypy==0.761;
     fi
   - pip install pytest
   - pip install coverage codecov
@@ -86,5 +74,16 @@ jobs:
         # Check for typos
         - sphinx-build -W -b spelling -d _build/doctrees . _build/spelling
 
+  fast_finish: true
+  exclude:
+    - python: "3.6"
+      env: TORCH_VER="1.0.1"
+    - python: "3.6"
+      env: TORCH_VER="1.2.0"
+    - python: "3.6"
+      env: TORCH_VER="1.3.0"
+    - python: "3.6"
+      env: TORCH_VER="1.4.0"
+
 notifications:
   email: false

examples/bert/data/download_glue_data.py

@@ -129,19 +129,21 @@ def main(arguments) -> None:
     for task in tasks:
         if task == 'MRPC':
             import subprocess
-            if not os.path.exists("data/MRPC"):
-                subprocess.run("mkdir data/MRPC", shell=True)
             # pylint: disable=line-too-long
+            if not os.path.exists("data/MRPC"):
+                subprocess.run("mkdir data/MRPC", shell=True, check=False)
             subprocess.run(
                 'wget -P data/MRPC/ https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt',
-                shell=True)
+                shell=True, check=False)
             subprocess.run(
                 'wget -P data/MRPC/ https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt',
-                shell=True)
+                shell=True, check=False)
             # pylint: enable=line-too-long
             format_mrpc(args.data_dir, args.path_to_mrpc)
-            subprocess.run('rm data/MRPC/msr_paraphrase_train.txt', shell=True)
-            subprocess.run('rm data/MRPC/msr_paraphrase_test.txt', shell=True)
+            subprocess.run('rm data/MRPC/msr_paraphrase_train.txt',
+                           shell=True, check=False)
+            subprocess.run('rm data/MRPC/msr_paraphrase_test.txt',
+                           shell=True, check=False)
         elif task == 'diagnostic':
             download_diagnostic(args.data_dir)
         else:

examples/gpt-2/gpt2_generate_main.py

@@ -15,6 +15,7 @@
 """
 import argparse
 import random
+import sys
 
 import numpy as np
 import torch
@@ -114,7 +115,7 @@ def _get_helper(start_tokens):
                     raw_text = input("Model input >>> ")
             except EOFError:
                 print("EOF entered, quitting.")
-                exit(0)
+                sys.exit()
 
             context_tokens = tokenizer.map_text_to_id(raw_text)
             context = torch.tensor(

examples/xlnet/xlnet_generation_main.py

@@ -15,6 +15,8 @@
 """
 
 import argparse
+import sys
+
 import torch
 
 import texar.torch as tx
@@ -142,7 +144,7 @@ def sample(text: str, length: int = 100, n_samples=3, **kwargs):
                        n_samples=batch_size)
             except EOFError:
                 print("EOF entered, quitting.")
-                exit(0)
+                sys.exit()
     else:
         # Generate samples from scratch
         for _ in range(nsamples // batch_size):

stubs/torch/__init__.pyi

@@ -9,6 +9,8 @@ from .storage import _StorageBase
 from .tensor import Tensor as TensorBase
 from .utils.hooks import RemovableHandle
 
+from . import backends as backends
+from . import cuda as cuda
 from . import optim as optim
 
 
@@ -105,6 +107,7 @@ double = float64 = _float64()
 short = int16 = _int16()
 long = int64 = _int64()
 uint8 = _uint8()
+int8 = _int8()
 float = float32 = _float32()
 int = int32 = _int32()
 

stubs/torch/backends/__init__.pyi

@@ -0,0 +1 @@
+from . import cudnn as cudnn

stubs/torch/backends/cudnn/__init__.pyi

@@ -0,0 +1,3 @@
+benchmark: bool = ...
+deterministic: bool = ...
+verbose: bool = ...

stubs/torch/cuda/__init__.pyi

@@ -0,0 +1,47 @@
+from typing import Optional, Tuple, Union
+import ctypes
+from .. import device as _device
+
+def is_available() -> bool: ...
+def init() -> None: ...
+
+class cudaStatus:
+    SUCCESS: int
+    ERROR_NOT_READY: int
+
+class CudaError:
+    def __init__(self, code: int) -> None: ...
+
+class _CudaDeviceProperties:
+    name: str
+    major: int
+    minor: int
+    multi_processor_count: int
+    total_memory: int
+    is_integrated: int
+    is_multi_gpu_board: int
+
+_device_t = Union[_device, int]
+
+def check_error(res: int) -> None: ...
+def device_count() -> int: ...
+def empty_cache() -> None: ...
+def synchronize(device: _device_t) -> None: ...
+def set_device(device: _device_t) -> None: ...
+def get_device_capability(device: Optional[_device_t]=...) -> Tuple[int, int]: ...
+def get_device_name(device: Optional[_device_t]=...) -> str: ...
+def get_device_properties(device: _device_t) -> _CudaDeviceProperties: ...
+def current_device() -> int: ...
+def memory_allocated(device: Optional[_device_t]=...) -> int: ...
+def max_memory_allocated(device: Optional[_device_t]=...) -> int: ...
+def reset_max_memory_allocated(device: Optional[_device_t]=...) -> None: ...
+def memory_cached(device: Optional[_device_t]=...) -> int: ...
+def max_memory_cached(device: Optional[_device_t]=...) -> int: ...
+def reset_max_memory_cached(device: Optional[_device_t]=...) -> None: ...
+def cudart() -> ctypes.CDLL: ...
+def find_cuda_windows_lib() -> Optional[ctypes.CDLL]: ...
+def set_rng_state(new_state): ...
+def get_rng_state(): ...
+
+def manual_seed(seed: int): ...
+def manual_seed_all(seed: int): ...

texar/torch/data/data/data_base.py

@@ -179,7 +179,7 @@ def __iter__(self) -> Iterator[Dict[str, RawExample]]:
         keys = list(self._sources.keys())
         iterator = zip(*[iter(source) for source in self._sources.values()])
         for values in iterator:
-            yield {key: value for key, value in zip(keys, values)}
+            yield dict(zip(keys, values))
 
     def __len__(self) -> int:
         return min(len(source) for source in self._sources.values())

texar/torch/data/data/text_data_base.py

@@ -131,8 +131,9 @@ def __getattr__(self, item):
 
     def _open_file(self, path: str) -> IO[str]:
         if self._compression_type == 'zlib':
-            f: IO[str] = io.TextIOWrapper(  # type: ignore
-                self._ZlibWrapper(open(path, 'rb')), encoding=self._encoding)
+            f: IO[str] = io.TextIOWrapper(
+                self._ZlibWrapper(open(path, 'rb')),  # type: ignore
+                encoding=self._encoding)
         elif self._compression_type == 'gzip':
             import gzip
             f = gzip.open(path, 'rt', encoding=self._encoding)

texar/torch/data/tokenizers/gpt2_tokenizer.py

@@ -218,8 +218,7 @@ def _bpe(self, token: str) -> str:
             word = new_word
             if len(word) == 1:
                 break
-            else:
-                pairs = get_pairs(word)
+            pairs = get_pairs(word)
         word = ' '.join(word)
         self.cache[token] = word
         return word

texar/torch/evals/bleu_moses.py

@@ -153,8 +153,8 @@ def corpus_bleu_moses(list_of_references: List[List[MaybeList[str]]],
         try:
             multi_bleu_ret = subprocess.check_output(
                 multi_bleu_cmd, stdin=hyp_input, stderr=subprocess.STDOUT)
-            multi_bleu_ret = multi_bleu_ret.decode("utf-8")
-            bleu_score = _parse_multi_bleu_ret(multi_bleu_ret, return_all)
+            multi_bleu_ret_ = multi_bleu_ret.decode("utf-8")
+            bleu_score = _parse_multi_bleu_ret(multi_bleu_ret_, return_all)
         except subprocess.CalledProcessError as error:
             if error.output is not None:
                 logging.warning("multi-bleu.perl returned non-zero exit code")

texar/torch/hyperparams.py

@@ -223,11 +223,10 @@ def _parse(hparams: Optional[Dict[str, Any]],
                 if allow_new_hparam:
                     parsed_hparams[name] = HParams._parse_value(value, name)
                     continue
-                else:
-                    raise ValueError(
-                        "Unknown hyperparameter: %s. Only hyperparameters "
-                        "named 'kwargs' hyperparameters can contain new "
-                        "entries undefined in default hyperparameters." % name)
+                raise ValueError(
+                    "Unknown hyperparameter: %s. Only hyperparameters "
+                    "named 'kwargs' hyperparameters can contain new "
+                    "entries undefined in default hyperparameters." % name)
 
             if value is None:
                 parsed_hparams[name] = HParams._parse_value(

texar/torch/losses/rewards.py

@@ -58,7 +58,7 @@ def discount_reward(reward: torch.Tensor,
 
     tensor_rank = reward.dim()
     if tensor_rank == 1:
-        disc_reward = _discount_reward_tensor_1d(  # type: ignore
+        disc_reward = _discount_reward_tensor_1d(
             reward, sequence_length, discount)
     elif tensor_rank == 2:
         disc_reward = _discount_reward_tensor_2d(
@@ -75,7 +75,7 @@ def discount_reward(reward: torch.Tensor,
 
 
 def _discount_reward_tensor_1d(reward: torch.Tensor,
-                               sequence_length: torch.LongTensor,
+                               sequence_length: Optional[torch.LongTensor],
                                discount: float = 1.) -> torch.Tensor:
     r"""Computes discounted reward.
 

texar/torch/modules/classifiers/conv_classifiers.py

@@ -163,7 +163,8 @@ def default_hparams() -> Dict[str, Any]:
 
     def forward(self,  # type:ignore
                 input: torch.Tensor,
-                sequence_length: Union[torch.LongTensor, List[int]] = None,
+                sequence_length: Optional[Union[torch.LongTensor,
+                                                List[int]]] = None,
                 dtype: Optional[torch.dtype] = None,
                 data_format: Optional[str] = None) \
             -> Tuple[torch.Tensor, torch.Tensor]:

texar/torch/modules/decoders/rnn_decoders.py

@@ -729,9 +729,12 @@ def forward(  # type: ignore
             if initial_state is not None:
                 state = state._replace(cell_state=initial_state)
 
-            sample_id, log_prob = self.beam_decode(  # type: ignore
+            end_token = kwargs.get('end_token')
+            assert isinstance(end_token, int)
+
+            sample_id, log_prob = self.beam_decode(
                 start_tokens=start_tokens,
-                end_token=kwargs.get('end_token'),
+                end_token=end_token,
                 initial_state=state,
                 beam_width=beam_width,
                 length_penalty=length_penalty,
@@ -761,8 +764,8 @@ def forward(  # type: ignore
         self._cell.init_batch()
 
         (outputs, final_state,
-         sequence_lengths) = self.dynamic_decode(  # type: ignore
-            helper, inputs, sequence_length, initial_state,
+         sequence_lengths) = self.dynamic_decode(
+            helper, inputs, sequence_length, initial_state,  # type: ignore
             max_decoding_length, impute_finished)
 
         # Release memory and memory_sequence_length in AttentionRNNDecoder

texar/torch/modules/networks/conv_networks.py

@@ -564,7 +564,8 @@ def _dropout_hparams():
 
     def forward(self,  # type: ignore
                 input: torch.Tensor,
-                sequence_length: Union[torch.LongTensor, List[int]] = None,
+                sequence_length: Optional[Union[torch.LongTensor,
+                                                List[int]]] = None,
                 dtype: Optional[torch.dtype] = None,
                 data_format: Optional[str] = None) -> torch.Tensor:
         r"""Feeds forward inputs through the network layers and returns outputs.

texar/torch/modules/networks/networks.py

@@ -97,7 +97,7 @@ def output_size(self) -> int:
                 size = size_ext
             if size is None:
                 break
-            elif size > 0:
+            if size > 0:
                 return size
             elif i == len(self._layers) - 1:
                 return -1

texar/torch/utils/average_recorder.py

@@ -176,7 +176,7 @@ def _to_dict(self, record: Record) -> Dict[ID, Scalar]:
         if isinstance(record, dict):
             record_dict = record
         elif isinstance(record, (list, tuple)):
-            record_dict = {i: vi for i, vi in enumerate(record)}
+            record_dict = dict(enumerate(record))
         else:
             record_dict = {self._default_metric_name: record}
         return record_dict

texar/torch/utils/utils.py

@@ -273,7 +273,6 @@ def sequence_mask(lengths: Union[torch.LongTensor, List[int]],
     size = lengths.size()
     row_vector = torch.arange(max_len, device=device, dtype=lengths.dtype).view(
         *([1] * len(size)), -1).expand(*size, max_len)
-    row_vector = row_vector
     mask = (row_vector < lengths.unsqueeze(-1)).to(device=device)
     if dtype is not None:
         mask = mask.to(dtype=dtype)