Bring back GCS ops.

.bazelversion

@@ -1 +1 @@
-3.1.0
+3.7.2

.github/workflows/api.yml

@@ -1,6 +1,9 @@
 name: API Compatibility
 
 on:
+ push:
+ branches:
+ - master
  schedule:
  - cron: "0 12 * * *"
 
@@ -13,12 +16,12 @@ jobs:
  name: macOS ${{ matrix.python }} + ${{ matrix.version }}
  runs-on: macos-latest
  strategy:
+ fail-fast: false
  matrix:
  python: ['3.8']
- version: ['tensorflow==2.4.0rc4:tensorflow-io-nightly', 'tf-nightly:tensorflow-io-nightly']
+ version: ['tensorflow==2.4.0:tensorflow-io-nightly', 'tf-nightly:tensorflow-io==0.17.0', 'tf-nightly:tensorflow-io-nightly']
  steps:
  - uses: actions/checkout@v2
- - uses: docker-practice/actions-setup-docker@v1
  - uses: actions/setup-python@v1
  with:
  python-version: ${{ matrix.python }}
@@ -29,9 +32,7 @@ jobs:
  - name: Setup macOS
  run: |
  set -x -e
- docker version
  bash -x -e tests/test_azure/start_azure.sh
- bash -x -e tests/test_aws/aws_test.sh
  - name: Test macOS
  run: |
  set -x -e
@@ -40,21 +41,23 @@ jobs:
  rm -rf tensorflow_io
  echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
  echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
- python -m pip install pytest-benchmark boto3
+ python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0
  python -m pip freeze
  python -c 'import tensorflow as tf; print(tf.version.VERSION)'
  python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
- python -m pytest -s -v tests/test_http_eager.py
- python -m pytest -s -v tests/test_s3_eager.py
+ python -m pytest -s -v tests/test_http.py
+ python -m pytest -s -v tests/test_s3.py
  python -m pytest -s -v tests/test_azure.py
+ python -m pytest -s -v tests/test_gcs.py
 
  linux:
  name: Linux ${{ matrix.python }} + ${{ matrix.version }}
  runs-on: ubuntu-20.04
  strategy:
+ fail-fast: false
  matrix:
  python: ['3.8']
- version: ['tensorflow==2.4.0rc4:tensorflow-io-nightly', 'tf-nightly:tensorflow-io-nightly']
+ version: ['tensorflow==2.4.0:tensorflow-io-nightly', 'tf-nightly:tensorflow-io==0.17.0', 'tf-nightly:tensorflow-io-nightly']
  steps:
  - uses: actions/checkout@v2
  - uses: actions/setup-python@v1
@@ -75,21 +78,23 @@ jobs:
  rm -rf tensorflow_io
  echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
  echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
- python -m pip install pytest-benchmark boto3
+ python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0
  python -m pip freeze
  python -c 'import tensorflow as tf; print(tf.version.VERSION)'
  python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
- python -m pytest -s -v tests/test_http_eager.py
- python -m pytest -s -v tests/test_s3_eager.py
+ python -m pytest -s -v tests/test_http.py
+ python -m pytest -s -v tests/test_s3.py
  python -m pytest -s -v tests/test_azure.py
+ if [[ "${{ matrix.version }}" != "tf-nightly:tensorflow-io==0.17.0" ]]; then python -m pytest -s -v tests/test_gcs.py ; fi
 
  windows:
  name: Windows ${{ matrix.python }} + ${{ matrix.version }}
  runs-on: windows-latest
  strategy:
+ fail-fast: false
  matrix:
  python: ['3.8']
- version: ['tensorflow==2.4.0rc4:tensorflow-io-nightly', 'tf-nightly:tensorflow-io-nightly']
+ version: ['tensorflow==2.4.0:tensorflow-io-nightly', 'tf-nightly:tensorflow-io==0.17.0', 'tf-nightly:tensorflow-io-nightly']
  steps:
  - uses: actions/checkout@v2
  - uses: actions/setup-python@v1
@@ -115,4 +120,4 @@ jobs:
  python -m pip freeze
  python -c 'import tensorflow as tf; print(tf.version.VERSION)'
  python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
- python -m pytest -s -v tests/test_http_eager.py -k remote
+ python -m pytest -s -v tests/test_http.py -k remote

.github/workflows/benchmarks.yml

@@ -0,0 +1,81 @@
+name: API Performance Benchmarks
+
+on:
+ push:
+ branches:
+ - master
+
+jobs:
+ macos:
+ name: macOS ${{ matrix.python }} + ${{ matrix.version }}
+ runs-on: macos-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python: ['3.8']
+ version: ['tensorflow==2.4.0:tensorflow-io-nightly', 'tensorflow==2.4.0:tensorflow-io']
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Setup macOS
+ run: |
+ set -x -e
+ python -m pip install -U wheel setuptools
+ python --version
+ - name: Benchmark on macOS
+ run: |
+ set -x -e
+ python --version
+ df -h
+ rm -rf tensorflow_io
+ echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
+ echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
+ python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0
+ python -m pip freeze
+ python -c 'import tensorflow as tf; print(tf.version.VERSION)'
+ python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
+ python -m pytest --benchmark-only -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \))
+
+ linux:
+ name: Linux ${{ matrix.python }} + ${{ matrix.version }}
+ runs-on: ubuntu-20.04
+ strategy:
+ fail-fast: false
+ matrix:
+ python: ['3.8']
+ version: ['tensorflow==2.4.0:tensorflow-io-nightly', 'tensorflow==2.4.0:tensorflow-io']
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ fetch-depth: 0
+ - uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Setup Linux
+ run: |
+ set -x -e
+ bash -x -e .github/workflows/build.space.sh
+ bash -x -e tests/test_sql/sql_test.sh
+ - name: Benchmark on Linux
+ run: |
+ set -x -e
+ python --version
+ df -h
+ rm -rf tensorflow_io
+ echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
+ echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
+ python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0
+ python -m pip freeze
+ python -c 'import tensorflow as tf; print(tf.version.VERSION)'
+ python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
+ python -m pytest --benchmark-only --benchmark-json benchmark.json -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \))
+ - name: Store benchmark result
+ uses: rhysd/github-action-benchmark@v1
+ with:
+ name: Tensorflow-IO Benchmarks
+ tool: 'pytest'
+ output-file-path: benchmark.json
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ auto-push: true

.github/workflows/build.wheel.sh

@@ -6,11 +6,11 @@ run_test() {
  entry=$1
  CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))')
  (cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl)
- $entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==0.16.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0
- (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_eager.py" \) \)))
- (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*_eager.py" ! \( -iname "test_bigquery_eager.py" \) \)))
- # GRPC and test_bigquery_eager tests have to be executed separately because of https://github.com/grpc/grpc/issues/20034
- (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_bigquery_eager.py" \)))
+ $entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1
+ (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*_v1.py" \)))
+ (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \)))
+ # GRPC and test_bigquery tests have to be executed separately because of https://github.com/grpc/grpc/issues/20034
+ (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_bigquery.py" \)))
 }
 
 PYTHON_VERSION=python