diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 536554a25..d2f929950 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: run: | set -x -e git log --pretty -1 - sudo python3 -m pip install -U numpy + sudo python3 -m pip install -U numpy black pyupgrade bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade black clang lint-docs: diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index 5981fe6fb..1db0308b7 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: | - sudo python3 -m pip install -U numpy pip + sudo python3 -m pip install -U numpy pip black pyupgrade bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- bazel - uses: reviewdog/action-suggester@v1 black: @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: | - sudo python3 -m pip install -U numpy pip + sudo python3 -m pip install -U numpy pip black pyupgrade bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- black - uses: reviewdog/action-suggester@v1 clang: @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: | - sudo python3 -m pip install -U numpy pip + sudo python3 -m pip install -U numpy pip black pyupgrade bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- clang - uses: reviewdog/action-suggester@v1 pyupgrade: @@ -39,6 +39,6 @@ jobs: steps: - uses: actions/checkout@v2 - run: | - sudo python3 -m pip install -U numpy pip + sudo python3 -m pip install -U numpy pip black pyupgrade bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- pyupgrade - uses: reviewdog/action-suggester@v1 diff --git a/WORKSPACE b/WORKSPACE index 2326a8539..02759f3e9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -91,27 +91,6 @@ switched_rules_by_language( grpc = True, ) -# Note rules_python is placed earlier as tensorflow's version is older -http_archive( - name = "rules_python", - sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz", - "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz", - ], -) - -load("@rules_python//python:pip.bzl", "pip_import") - -pip_import( - name = "lint_dependencies", - requirements = "//tools/lint:requirements.txt", -) - -load("@lint_dependencies//:requirements.bzl", "pip_install") - -pip_install() - http_archive( name = "org_tensorflow", sha256 = "e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc", diff --git a/tensorflow_io/python/experimental/filter_ops.py b/tensorflow_io/python/experimental/filter_ops.py index 7646fbd88..a7ad0a4ad 100644 --- a/tensorflow_io/python/experimental/filter_ops.py +++ b/tensorflow_io/python/experimental/filter_ops.py @@ -188,8 +188,8 @@ def gabor( 1.0 / np.pi * np.sqrt(np.log(2) / 2.0) - * (2.0 ** b + 1) - / (2.0 ** b - 1), + * (2.0**b + 1) + / (2.0**b - 1), dtype.real_dtype, ) / freq @@ -220,7 +220,7 @@ def gabor( rotx = y * tf.math.sin(theta) + x * tf.math.cos(theta) roty = y * tf.math.cos(theta) - x * tf.math.sin(theta) - g = tf.math.exp(-0.5 * (rotx ** 2 / sigma_x ** 2 + roty ** 2 / sigma_y ** 2)) + g = tf.math.exp(-0.5 * (rotx**2 / sigma_x**2 + roty**2 / sigma_y**2)) g = g / (2 * np.pi * sigma_x * sigma_y) g = tf.cast(g, dtype) * tf.exp( tf.cast(1j, dtype) * tf.cast(2 * np.pi * freq * rotx + offset, dtype) diff --git a/tensorflow_io/python/experimental/openexr_io_tensor_ops.py b/tensorflow_io/python/experimental/openexr_io_tensor_ops.py index a36e29c8f..f7d3f94a9 100644 --- a/tensorflow_io/python/experimental/openexr_io_tensor_ops.py +++ b/tensorflow_io/python/experimental/openexr_io_tensor_ops.py @@ -44,7 +44,7 @@ def __init__(self, filename, internal=False): shapes, dtypes, channels = core_ops.io_decode_exr_info(data) parts = [] index = 0 - for (shape, dtypes, channels) in zip( + for shape, dtypes, channels in zip( shapes.numpy(), dtypes.numpy(), channels.numpy() ): # Remove trailing 0 from dtypes diff --git a/tensorflow_io/python/experimental/serialization_ops.py b/tensorflow_io/python/experimental/serialization_ops.py index 4fa870592..cb311dc7f 100644 --- a/tensorflow_io/python/experimental/serialization_ops.py +++ b/tensorflow_io/python/experimental/serialization_ops.py @@ -19,6 +19,7 @@ import tensorflow as tf from tensorflow_io.python.ops import core_ops + # _NamedTensorSpec allows adding a `named` key while traversing, # so that it is possible to build up the `/R/Foo` JSON Pointers. class _NamedTensorSpec(tf.TensorSpec): diff --git a/tensorflow_io/python/ops/arrow_io_tensor_ops.py b/tensorflow_io/python/ops/arrow_io_tensor_ops.py index 7ef29a6f1..a79a95d9a 100644 --- a/tensorflow_io/python/ops/arrow_io_tensor_ops.py +++ b/tensorflow_io/python/ops/arrow_io_tensor_ops.py @@ -83,7 +83,6 @@ def _extract_table_arrays(table): # Pad buffer addrs, sizes and array lengths so inputs are rectangular num_columns = len(array_buffer_sizes) for i in range(num_columns): - # pad chunk list with empty lists that will be padded with null bufs if len(array_buffer_sizes[i]) < max_num_chunks: array_buffer_sizes[i].extend( @@ -94,7 +93,6 @@ def _extract_table_arrays(table): num_chunks = len(array_buffer_sizes[i]) for j in range(num_chunks): - # pad buffer addr, size, and array length lists if len(array_buffer_sizes[i][j]) < max_num_bufs: array_buffer_sizes[i][j].extend( @@ -140,7 +138,6 @@ def wrap_func(inp): def init_resource_op(self, table): """Initialize the resource op with a pyarrow.Table""" with tf.name_scope("ArrowIOTensor") as scope: - # Serialize the schema to send to the kernel schema_buffer = table.schema.serialize() @@ -241,7 +238,6 @@ class ArrowIOTensor(io_tensor_ops._TableIOTensor): # pylint: disable=protected- # Constructor (private) # ============================================================================= def __init__(self, table, spec=None, internal=False): - # Get or build a new ArrowIOResource if isinstance(table, ArrowIOResource): arrow_resource = table @@ -292,7 +288,7 @@ def __init__(self, table, spec=None, internal=False): ] shapes = [] - for (column_index, column) in columns: + for column_index, column in columns: shape, _ = core_ops.io_arrow_readable_spec( resource, column_index, column ) @@ -306,7 +302,7 @@ def __init__(self, table, spec=None, internal=False): ] elements = [] - for ((column_index, column), entry, shape) in zip(columns, entries, shapes): + for (column_index, column), entry, shape in zip(columns, entries, shapes): function = _ArrowIOTensorComponentFunction( core_ops.io_arrow_readable_read, resource, diff --git a/tensorflow_io/python/ops/avro_dataset_ops.py b/tensorflow_io/python/ops/avro_dataset_ops.py index a606569eb..f02848785 100644 --- a/tensorflow_io/python/ops/avro_dataset_ops.py +++ b/tensorflow_io/python/ops/avro_dataset_ops.py @@ -75,7 +75,7 @@ def __init__(self, filename, schema, columns=None, internal=True): ) columns_function.append(function) - for (column, function) in zip(columns, columns_function): + for column, function in zip(columns, columns_function): column_dataset = tf.compat.v2.data.Dataset.range( 0, sys.maxsize, capacity ) diff --git a/tensorflow_io/python/ops/data_ops.py b/tensorflow_io/python/ops/data_ops.py index ef5361460..78ebd81f0 100644 --- a/tensorflow_io/python/ops/data_ops.py +++ b/tensorflow_io/python/ops/data_ops.py @@ -16,6 +16,7 @@ import tensorflow as tf + # Note: BaseDataset could be used by Dataset implementations # that does not utilize DataInput implementation. class BaseDataset(tf.compat.v2.data.Dataset): diff --git a/tensorflow_io/python/ops/json_dataset_ops.py b/tensorflow_io/python/ops/json_dataset_ops.py index b9ad0115e..5dd4f0163 100644 --- a/tensorflow_io/python/ops/json_dataset_ops.py +++ b/tensorflow_io/python/ops/json_dataset_ops.py @@ -75,7 +75,7 @@ def __init__(self, filename, columns=None, mode=None, internal=True): ) columns_function.append(function) - for (column, function) in zip(columns, columns_function): + for column, function in zip(columns, columns_function): column_dataset = tf.compat.v2.data.Dataset.range( 0, sys.maxsize, capacity ) diff --git a/tensorflow_io/python/ops/orc_dataset_ops.py b/tensorflow_io/python/ops/orc_dataset_ops.py index 6a8515e72..42309da5a 100644 --- a/tensorflow_io/python/ops/orc_dataset_ops.py +++ b/tensorflow_io/python/ops/orc_dataset_ops.py @@ -69,7 +69,7 @@ def __init__(self, filename, columns=None, internal=True, **kwargs): ) columns_function.append(function) - for (column, function) in zip(columns, columns_function): + for column, function in zip(columns, columns_function): column_dataset = tf.compat.v2.data.Dataset.range( 0, sys.maxsize, capacity ) diff --git a/tensorflow_io/python/ops/prometheus_dataset_ops.py b/tensorflow_io/python/ops/prometheus_dataset_ops.py index 6fb277c4f..b9254e888 100644 --- a/tensorflow_io/python/ops/prometheus_dataset_ops.py +++ b/tensorflow_io/python/ops/prometheus_dataset_ops.py @@ -56,6 +56,7 @@ def __init__( ) spec_by_job[instance] = spec_by_instance spec[job] = spec_by_job + # Map spec to entries of 3 tuple (job, instance, name) class MetricEntry: def __init__(self, job, instance, name): diff --git a/tests/test_arrow.py b/tests/test_arrow.py index 68f442f33..6ff64a50b 100644 --- a/tests/test_arrow.py +++ b/tests/test_arrow.py @@ -378,10 +378,8 @@ def evaluate_result(value): # Iterate over the dataset for results in dataset: - # For batches, iterate over each row in batch or remainder at end for result_idx in range(batch_size or 1): - # Get a single row value if batch_size is None: value = [r.numpy() for r in results] diff --git a/tests/test_atds_avro/utils/stat_t_test_cli.py b/tests/test_atds_avro/utils/stat_t_test_cli.py index be5c1f976..0121ebbb4 100644 --- a/tests/test_atds_avro/utils/stat_t_test_cli.py +++ b/tests/test_atds_avro/utils/stat_t_test_cli.py @@ -139,13 +139,13 @@ def run_welchs_ttest(stat1, stat2, alpha, faster): df1 = n1 - 1 # degree of freedom of stat1 df2 = n2 - 1 # degree of freedom of stat2 - sample_v1 = s1 ** 2 / n1 # biased estimated sample variance of stat1 - sample_v2 = s2 ** 2 / n2 # biased estimated sample variance of stat2 + sample_v1 = s1**2 / n1 # biased estimated sample variance of stat1 + sample_v2 = s2**2 / n2 # biased estimated sample variance of stat2 biased_variance = np.sqrt(sample_v1 + sample_v2) # degree of freedom df = (sample_v1 + sample_v2) ** 2 / ( - sample_v1 ** 2 / (df1) + sample_v2 ** 2 / (df2) + sample_v1**2 / (df1) + sample_v2**2 / (df2) ) mean_delta = m1 - m2 diff --git a/tests/test_audio.py b/tests/test_audio.py index 8fa634300..dbada860c 100644 --- a/tests/test_audio.py +++ b/tests/test_audio.py @@ -77,7 +77,7 @@ def test_audio_io_tensor_24(audio_data_24, io_tensor_func): for step in [1, 100, 101, 200, 501, 600, 1001, 2000, 5001]: indices = list(range(0, 14336, step)) # TODO: -1 vs. 14336 might need fix - for (start, stop) in zip(indices, indices[1:] + [14336]): + for start, stop in zip(indices, indices[1:] + [14336]): audio_tensor_value = audio_tensor[start:stop] audio_value_value = audio_value[start:stop] assert audio_tensor_value.shape == audio_value_value.shape @@ -139,7 +139,7 @@ def test_audio_io_tensor(audio_data, io_tensor_func): for step in [1, 100, 101, 200, 501, 600, 1001, 2000, 5001]: indices = list(range(0, 5760, step)) # TODO: -1 vs. 5760 might need fix - for (start, stop) in zip(indices, indices[1:] + [5760]): + for start, stop in zip(indices, indices[1:] + [5760]): audio_tensor_value = audio_tensor[start:stop] audio_value_value = audio_value[start:stop] assert audio_tensor_value.shape == audio_value_value.shape @@ -218,7 +218,7 @@ def func(filename, position): dataset = dataset.map(func, num_parallel_calls=num_parallel_calls) item = 0 - for (data, rate) in dataset: + for data, rate in dataset: assert audio_rate == rate assert data.shape == (100, 1) position = 1000 if item == 0 else 2000 diff --git a/tests/test_azure.py b/tests/test_azure.py index b89f5e556..3fef84edf 100644 --- a/tests/test_azure.py +++ b/tests/test_azure.py @@ -206,7 +206,6 @@ class AZFSTest(tf.test.TestCase, AZFSTestBase): """Run tests for azfs backend using account key authentication.""" def __init__(self, methodName="runTest"): # pylint: disable=invalid-name - self.account = "devstoreaccount1" self.container = "aztest" self.path_root = "az://" + os.path.join(self.account, self.container) diff --git a/tests/test_http.py b/tests/test_http.py index 2cb5bf0af..81b828a78 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -77,7 +77,7 @@ def test_gfile_read(local_content, remote_filename): assert remote_gfile.size() == len(local_content) offset_start = list(range(0, len(local_content), 100)) offset_stop = offset_start[1:] + [len(local_content)] - for (start, stop) in zip(offset_start, offset_stop): + for start, stop in zip(offset_start, offset_stop): assert remote_gfile.read(100) == local_content[start:stop] diff --git a/tests/test_json.py b/tests/test_json.py index bccab123b..5e393a9b7 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -61,7 +61,7 @@ def test_io_tensor_json_recods_mode(): dataset = tf.data.Dataset.zip((feature_dataset, label_dataset)) i = 0 - for (j_x, j_y) in dataset: + for j_x, j_y in dataset: v_x = x_test[i] v_y = y_test[i] for index, x in enumerate(j_x): @@ -123,7 +123,7 @@ def test_io_tensor_json(): dataset = tf.data.Dataset.zip((feature_dataset, label_dataset)) i = 0 - for (j_x, j_y) in dataset: + for j_x, j_y in dataset: v_x = x_test[i] v_y = y_test[i] for index, x in enumerate(j_x): @@ -158,7 +158,7 @@ def test_json_dataset(): dataset = tf.data.Dataset.zip((feature_dataset, label_dataset)) i = 0 - for (j_x, j_y) in dataset: + for j_x, j_y in dataset: v_x = x_test[i] v_y = y_test[i] for index, x in enumerate(j_x): diff --git a/tests/test_ossfs.py b/tests/test_ossfs.py index 4c2470e79..1bad9e30c 100644 --- a/tests/test_ossfs.py +++ b/tests/test_ossfs.py @@ -359,7 +359,7 @@ def test_walk_in_order(self): all_dirs = [] all_subdirs = [] all_files = [] - for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=True): + for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=True): all_dirs.append(w_dir) all_subdirs.append(w_subdirs) all_files.append(w_files) @@ -388,7 +388,7 @@ def test_walk_post_order(self): all_dirs = [] all_subdirs = [] all_files = [] - for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=False): + for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=False): all_dirs.append(w_dir) all_subdirs.append(w_subdirs) all_files.append(w_files) @@ -416,7 +416,7 @@ def test_walk_failure(self): all_dirs = [] all_subdirs = [] all_files = [] - for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=False): + for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=False): all_dirs.append(w_dir) all_subdirs.append(w_subdirs) all_files.append(w_files) diff --git a/tests/test_parse_avro.py b/tests/test_parse_avro.py index 611e1e272..aca577647 100644 --- a/tests/test_parse_avro.py +++ b/tests/test_parse_avro.py @@ -223,7 +223,6 @@ def _assert_equal(expected, actual): _assert_equal(expected, actual) def _verify_output(self, expected_data, actual_dataset): - next_data = iter(actual_dataset) for expected in expected_data: @@ -286,7 +285,6 @@ def test_inval_num_parallel_calls(self): ) with pytest.raises(ValueError): - dataset_a = tfio.experimental.columnar.AvroRecordDataset( filenames=filenames, num_parallel_reads=NUM_PARALLEL_READS, @@ -295,7 +293,6 @@ def test_inval_num_parallel_calls(self): ) with pytest.raises(ValueError): - dataset_b = tfio.experimental.columnar.AvroRecordDataset( filenames=filenames, num_parallel_reads=NUM_PARALLEL_READS, diff --git a/tests/test_pulsar.py b/tests/test_pulsar.py index 30e59bad7..9fc8d8851 100644 --- a/tests/test_pulsar.py +++ b/tests/test_pulsar.py @@ -66,7 +66,7 @@ def test_pulsar_keyed_messages(): timeout=default_pulsar_timeout, ) kv = dict() - for (msg, key) in dataset: + for msg, key in dataset: kv.setdefault(key.numpy().decode(), []).append(msg.numpy()) assert kv["K0"] == [("D" + str(i)).encode() for i in range(0, 6, 2)] assert kv["K1"] == [("D" + str(i)).encode() for i in range(1, 6, 2)] @@ -232,7 +232,7 @@ def test_pulsar_write_keyed_messages(): timeout=default_pulsar_timeout, ) kv = dict() - for (msg, key) in dataset: + for msg, key in dataset: kv.setdefault(key.numpy().decode(), []).append(msg.numpy()) assert kv["0"] == [("msg-" + str(i)).encode() for i in range(0, 10, 3)] assert kv["1"] == [("msg-" + str(i)).encode() for i in range(1, 10, 3)] diff --git a/tests/test_text/stdin_test.py b/tests/test_text/stdin_test.py index 9e344b709..34715719d 100644 --- a/tests/test_text/stdin_test.py +++ b/tests/test_text/stdin_test.py @@ -33,5 +33,5 @@ def f(v): text_dataset = text_io.TextDataset("file://-").map(f) -for (frame_number_value, ip_dst_value, ip_proto_value) in text_dataset: +for frame_number_value, ip_dst_value, ip_proto_value in text_dataset: print(ip_dst_value.numpy()) diff --git a/tools/lint/BUILD b/tools/lint/BUILD index 06e8c8ff7..3cc382f2e 100644 --- a/tools/lint/BUILD +++ b/tools/lint/BUILD @@ -1,5 +1,4 @@ load("defs.bzl", "lint") -load("@lint_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) @@ -20,8 +19,6 @@ py_binary( srcs = ["pyupgrade_python.py"], main = "pyupgrade_python.py", deps = [ - requirement("pyupgrade"), - requirement("tokenize-rt"), ], ) @@ -41,14 +38,6 @@ py_binary( srcs = ["black_python.py"], main = "black_python.py", deps = [ - requirement("black"), - requirement("click"), - requirement("mypy-extensions"), - requirement("pathspec"), - requirement("platformdirs"), - requirement("regex"), - requirement("tomli"), - requirement("typing-extensions"), ], )