Remove rules_python dependency (#1810)

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
  run: |
  set -x -e
  git log --pretty -1
- sudo python3 -m pip install -U numpy
+ sudo python3 -m pip install -U numpy black pyupgrade
  bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade black clang
 
  lint-docs:

.github/workflows/reviewdog.yml

@@ -12,7 +12,7 @@ jobs:
  steps:
  - uses: actions/checkout@v2
  - run: |
- sudo python3 -m pip install -U numpy pip
+ sudo python3 -m pip install -U numpy pip black pyupgrade
  bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- bazel
  - uses: reviewdog/action-suggester@v1
  black:
@@ -21,7 +21,7 @@ jobs:
  steps:
  - uses: actions/checkout@v2
  - run: |
- sudo python3 -m pip install -U numpy pip
+ sudo python3 -m pip install -U numpy pip black pyupgrade
  bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- black
  - uses: reviewdog/action-suggester@v1
  clang:
@@ -30,7 +30,7 @@ jobs:
  steps:
  - uses: actions/checkout@v2
  - run: |
- sudo python3 -m pip install -U numpy pip
+ sudo python3 -m pip install -U numpy pip black pyupgrade
  bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- clang
  - uses: reviewdog/action-suggester@v1
  pyupgrade:
@@ -39,6 +39,6 @@ jobs:
  steps:
  - uses: actions/checkout@v2
  - run: |
- sudo python3 -m pip install -U numpy pip
+ sudo python3 -m pip install -U numpy pip black pyupgrade
  bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- pyupgrade
  - uses: reviewdog/action-suggester@v1

WORKSPACE

@@ -91,27 +91,6 @@ switched_rules_by_language(
  grpc = True,
 )
 
-# Note rules_python is placed earlier as tensorflow's version is older
-http_archive(
- name = "rules_python",
- sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
- urls = [
- "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
- "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
- ],
-)
-
-load("@rules_python//python:pip.bzl", "pip_import")
-
-pip_import(
- name = "lint_dependencies",
- requirements = "//tools/lint:requirements.txt",
-)
-
-load("@lint_dependencies//:requirements.bzl", "pip_install")
-
-pip_install()
-
 http_archive(
  name = "org_tensorflow",
  sha256 = "e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc",

tensorflow_io/python/experimental/filter_ops.py

@@ -188,8 +188,8 @@ def gabor(
  1.0
  / np.pi
  * np.sqrt(np.log(2) / 2.0)
- * (2.0 ** b + 1)
- / (2.0 ** b - 1),
+ * (2.0**b + 1)
+ / (2.0**b - 1),
  dtype.real_dtype,
  )
  / freq
@@ -220,7 +220,7 @@ def gabor(
  rotx = y * tf.math.sin(theta) + x * tf.math.cos(theta)
  roty = y * tf.math.cos(theta) - x * tf.math.sin(theta)
 
- g = tf.math.exp(-0.5 * (rotx ** 2 / sigma_x ** 2 + roty ** 2 / sigma_y ** 2))
+ g = tf.math.exp(-0.5 * (rotx**2 / sigma_x**2 + roty**2 / sigma_y**2))
  g = g / (2 * np.pi * sigma_x * sigma_y)
  g = tf.cast(g, dtype) * tf.exp(
  tf.cast(1j, dtype) * tf.cast(2 * np.pi * freq * rotx + offset, dtype)

tensorflow_io/python/experimental/openexr_io_tensor_ops.py

@@ -44,7 +44,7 @@ def __init__(self, filename, internal=False):
  shapes, dtypes, channels = core_ops.io_decode_exr_info(data)
  parts = []
  index = 0
- for (shape, dtypes, channels) in zip(
+ for shape, dtypes, channels in zip(
  shapes.numpy(), dtypes.numpy(), channels.numpy()
  ):
  # Remove trailing 0 from dtypes

tensorflow_io/python/experimental/serialization_ops.py

@@ -19,6 +19,7 @@
 import tensorflow as tf
 from tensorflow_io.python.ops import core_ops
 
+
 # _NamedTensorSpec allows adding a `named` key while traversing,
 # so that it is possible to build up the `/R/Foo` JSON Pointers.
 class _NamedTensorSpec(tf.TensorSpec):

tensorflow_io/python/ops/arrow_io_tensor_ops.py

@@ -83,7 +83,6 @@ def _extract_table_arrays(table):
  # Pad buffer addrs, sizes and array lengths so inputs are rectangular
  num_columns = len(array_buffer_sizes)
  for i in range(num_columns):
-
  # pad chunk list with empty lists that will be padded with null bufs
  if len(array_buffer_sizes[i]) < max_num_chunks:
  array_buffer_sizes[i].extend(
@@ -94,7 +93,6 @@ def _extract_table_arrays(table):
 
  num_chunks = len(array_buffer_sizes[i])
  for j in range(num_chunks):
-
  # pad buffer addr, size, and array length lists
  if len(array_buffer_sizes[i][j]) < max_num_bufs:
  array_buffer_sizes[i][j].extend(
@@ -140,7 +138,6 @@ def wrap_func(inp):
  def init_resource_op(self, table):
  """Initialize the resource op with a pyarrow.Table"""
  with tf.name_scope("ArrowIOTensor") as scope:
-
  # Serialize the schema to send to the kernel
  schema_buffer = table.schema.serialize()
 
@@ -241,7 +238,6 @@ class ArrowIOTensor(io_tensor_ops._TableIOTensor): # pylint: disable=protected-
  # Constructor (private)
  # =============================================================================
  def __init__(self, table, spec=None, internal=False):
-
  # Get or build a new ArrowIOResource
  if isinstance(table, ArrowIOResource):
  arrow_resource = table
@@ -292,7 +288,7 @@ def __init__(self, table, spec=None, internal=False):
  ]
 
  shapes = []
- for (column_index, column) in columns:
+ for column_index, column in columns:
  shape, _ = core_ops.io_arrow_readable_spec(
  resource, column_index, column
  )
@@ -306,7 +302,7 @@ def __init__(self, table, spec=None, internal=False):
  ]
 
  elements = []
- for ((column_index, column), entry, shape) in zip(columns, entries, shapes):
+ for (column_index, column), entry, shape in zip(columns, entries, shapes):
  function = _ArrowIOTensorComponentFunction(
  core_ops.io_arrow_readable_read,
  resource,

tensorflow_io/python/ops/avro_dataset_ops.py

@@ -75,7 +75,7 @@ def __init__(self, filename, schema, columns=None, internal=True):
  )
  columns_function.append(function)
 
- for (column, function) in zip(columns, columns_function):
+ for column, function in zip(columns, columns_function):
  column_dataset = tf.compat.v2.data.Dataset.range(
  0, sys.maxsize, capacity
  )

tensorflow_io/python/ops/data_ops.py

@@ -16,6 +16,7 @@
 
 import tensorflow as tf
 
+
 # Note: BaseDataset could be used by Dataset implementations
 # that does not utilize DataInput implementation.
 class BaseDataset(tf.compat.v2.data.Dataset):

tensorflow_io/python/ops/json_dataset_ops.py

@@ -75,7 +75,7 @@ def __init__(self, filename, columns=None, mode=None, internal=True):
  )
  columns_function.append(function)
 
- for (column, function) in zip(columns, columns_function):
+ for column, function in zip(columns, columns_function):
  column_dataset = tf.compat.v2.data.Dataset.range(
  0, sys.maxsize, capacity
  )

tensorflow_io/python/ops/orc_dataset_ops.py

@@ -69,7 +69,7 @@ def __init__(self, filename, columns=None, internal=True, **kwargs):
  )
  columns_function.append(function)
 
- for (column, function) in zip(columns, columns_function):
+ for column, function in zip(columns, columns_function):
  column_dataset = tf.compat.v2.data.Dataset.range(
  0, sys.maxsize, capacity
  )

tensorflow_io/python/ops/prometheus_dataset_ops.py

@@ -56,6 +56,7 @@ def __init__(
  )
  spec_by_job[instance] = spec_by_instance
  spec[job] = spec_by_job
+
  # Map spec to entries of 3 tuple (job, instance, name)
  class MetricEntry:
  def __init__(self, job, instance, name):

tests/test_arrow.py

@@ -378,10 +378,8 @@ def evaluate_result(value):
 
  # Iterate over the dataset
  for results in dataset:
-
  # For batches, iterate over each row in batch or remainder at end
  for result_idx in range(batch_size or 1):
-
  # Get a single row value
  if batch_size is None:
  value = [r.numpy() for r in results]

tests/test_atds_avro/utils/stat_t_test_cli.py

@@ -139,13 +139,13 @@ def run_welchs_ttest(stat1, stat2, alpha, faster):
  df1 = n1 - 1 # degree of freedom of stat1
  df2 = n2 - 1 # degree of freedom of stat2
 
- sample_v1 = s1 ** 2 / n1 # biased estimated sample variance of stat1
- sample_v2 = s2 ** 2 / n2 # biased estimated sample variance of stat2
+ sample_v1 = s1**2 / n1 # biased estimated sample variance of stat1
+ sample_v2 = s2**2 / n2 # biased estimated sample variance of stat2
 
  biased_variance = np.sqrt(sample_v1 + sample_v2)
  # degree of freedom
  df = (sample_v1 + sample_v2) ** 2 / (
- sample_v1 ** 2 / (df1) + sample_v2 ** 2 / (df2)
+ sample_v1**2 / (df1) + sample_v2**2 / (df2)
  )
 
  mean_delta = m1 - m2

tests/test_audio.py

@@ -77,7 +77,7 @@ def test_audio_io_tensor_24(audio_data_24, io_tensor_func):
  for step in [1, 100, 101, 200, 501, 600, 1001, 2000, 5001]:
  indices = list(range(0, 14336, step))
  # TODO: -1 vs. 14336 might need fix
- for (start, stop) in zip(indices, indices[1:] + [14336]):
+ for start, stop in zip(indices, indices[1:] + [14336]):
  audio_tensor_value = audio_tensor[start:stop]
  audio_value_value = audio_value[start:stop]
  assert audio_tensor_value.shape == audio_value_value.shape
@@ -139,7 +139,7 @@ def test_audio_io_tensor(audio_data, io_tensor_func):
  for step in [1, 100, 101, 200, 501, 600, 1001, 2000, 5001]:
  indices = list(range(0, 5760, step))
  # TODO: -1 vs. 5760 might need fix
- for (start, stop) in zip(indices, indices[1:] + [5760]):
+ for start, stop in zip(indices, indices[1:] + [5760]):
  audio_tensor_value = audio_tensor[start:stop]
  audio_value_value = audio_value[start:stop]
  assert audio_tensor_value.shape == audio_value_value.shape
@@ -218,7 +218,7 @@ def func(filename, position):
  dataset = dataset.map(func, num_parallel_calls=num_parallel_calls)
 
  item = 0
- for (data, rate) in dataset:
+ for data, rate in dataset:
  assert audio_rate == rate
  assert data.shape == (100, 1)
  position = 1000 if item == 0 else 2000

tests/test_azure.py

@@ -206,7 +206,6 @@ class AZFSTest(tf.test.TestCase, AZFSTestBase):
  """Run tests for azfs backend using account key authentication."""
 
  def __init__(self, methodName="runTest"): # pylint: disable=invalid-name
-
  self.account = "devstoreaccount1"
  self.container = "aztest"
  self.path_root = "az://" + os.path.join(self.account, self.container)

tests/test_http.py

@@ -77,7 +77,7 @@ def test_gfile_read(local_content, remote_filename):
  assert remote_gfile.size() == len(local_content)
  offset_start = list(range(0, len(local_content), 100))
  offset_stop = offset_start[1:] + [len(local_content)]
- for (start, stop) in zip(offset_start, offset_stop):
+ for start, stop in zip(offset_start, offset_stop):
  assert remote_gfile.read(100) == local_content[start:stop]
 
 

tests/test_json.py

@@ -61,7 +61,7 @@ def test_io_tensor_json_recods_mode():
  dataset = tf.data.Dataset.zip((feature_dataset, label_dataset))
 
  i = 0
- for (j_x, j_y) in dataset:
+ for j_x, j_y in dataset:
  v_x = x_test[i]
  v_y = y_test[i]
  for index, x in enumerate(j_x):
@@ -123,7 +123,7 @@ def test_io_tensor_json():
  dataset = tf.data.Dataset.zip((feature_dataset, label_dataset))
 
  i = 0
- for (j_x, j_y) in dataset:
+ for j_x, j_y in dataset:
  v_x = x_test[i]
  v_y = y_test[i]
  for index, x in enumerate(j_x):
@@ -158,7 +158,7 @@ def test_json_dataset():
  dataset = tf.data.Dataset.zip((feature_dataset, label_dataset))
 
  i = 0
- for (j_x, j_y) in dataset:
+ for j_x, j_y in dataset:
  v_x = x_test[i]
  v_y = y_test[i]
  for index, x in enumerate(j_x):

tests/test_ossfs.py

@@ -359,7 +359,7 @@ def test_walk_in_order(self):
  all_dirs = []
  all_subdirs = []
  all_files = []
- for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=True):
+ for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=True):
  all_dirs.append(w_dir)
  all_subdirs.append(w_subdirs)
  all_files.append(w_files)
@@ -388,7 +388,7 @@ def test_walk_post_order(self):
  all_dirs = []
  all_subdirs = []
  all_files = []
- for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=False):
+ for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=False):
  all_dirs.append(w_dir)
  all_subdirs.append(w_subdirs)
  all_files.append(w_files)
@@ -416,7 +416,7 @@ def test_walk_failure(self):
  all_dirs = []
  all_subdirs = []
  all_files = []
- for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=False):
+ for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=False):
  all_dirs.append(w_dir)
  all_subdirs.append(w_subdirs)
  all_files.append(w_files)

tests/test_parse_avro.py

@@ -223,7 +223,6 @@ def _assert_equal(expected, actual):
  _assert_equal(expected, actual)
 
  def _verify_output(self, expected_data, actual_dataset):
-
  next_data = iter(actual_dataset)
 
  for expected in expected_data:
@@ -286,7 +285,6 @@ def test_inval_num_parallel_calls(self):
  )
 
  with pytest.raises(ValueError):
-
  dataset_a = tfio.experimental.columnar.AvroRecordDataset(
  filenames=filenames,
  num_parallel_reads=NUM_PARALLEL_READS,
@@ -295,7 +293,6 @@ def test_inval_num_parallel_calls(self):
  )
 
  with pytest.raises(ValueError):
-
  dataset_b = tfio.experimental.columnar.AvroRecordDataset(
  filenames=filenames,
  num_parallel_reads=NUM_PARALLEL_READS,

tests/test_pulsar.py

@@ -66,7 +66,7 @@ def test_pulsar_keyed_messages():
  timeout=default_pulsar_timeout,
  )
  kv = dict()
- for (msg, key) in dataset:
+ for msg, key in dataset:
  kv.setdefault(key.numpy().decode(), []).append(msg.numpy())
  assert kv["K0"] == [("D" + str(i)).encode() for i in range(0, 6, 2)]
  assert kv["K1"] == [("D" + str(i)).encode() for i in range(1, 6, 2)]
@@ -232,7 +232,7 @@ def test_pulsar_write_keyed_messages():
  timeout=default_pulsar_timeout,
  )
  kv = dict()
- for (msg, key) in dataset:
+ for msg, key in dataset:
  kv.setdefault(key.numpy().decode(), []).append(msg.numpy())
  assert kv["0"] == [("msg-" + str(i)).encode() for i in range(0, 10, 3)]
  assert kv["1"] == [("msg-" + str(i)).encode() for i in range(1, 10, 3)]

tests/test_text/stdin_test.py

@@ -33,5 +33,5 @@ def f(v):
 
 text_dataset = text_io.TextDataset("file://-").map(f)
 
-for (frame_number_value, ip_dst_value, ip_proto_value) in text_dataset:
+for frame_number_value, ip_dst_value, ip_proto_value in text_dataset:
  print(ip_dst_value.numpy())