fix gsheet unit tests

perfmetrics/scripts/testing_on_gke/examples/dlio/run_tests.py

@@ -30,8 +30,8 @@
 
 # local imports from other directories
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'utils'))
-from run_tests_common import escape_commas_in_string, parse_args, run_command, add_iam_role_for_buckets
-from utils import UnknownMachineTypeError, resource_limits
+from run_tests_common import escape_commas_in_string, parse_args, add_iam_role_for_buckets
+from utils import UnknownMachineTypeError, resource_limits, run_command
 
 # local imports from same directory
 import dlio_workload

perfmetrics/scripts/testing_on_gke/examples/fio/run_tests.py

@@ -29,8 +29,8 @@
 
 # local imports from other directories
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'utils'))
-from run_tests_common import escape_commas_in_string, parse_args, run_command, add_iam_role_for_buckets
-from utils import UnknownMachineTypeError, resource_limits
+from run_tests_common import escape_commas_in_string, parse_args, add_iam_role_for_buckets
+from utils import UnknownMachineTypeError, resource_limits, run_command
 
 # local imports from same directory
 import fio_workload

perfmetrics/scripts/testing_on_gke/examples/utils/gsheet.py

@@ -17,7 +17,7 @@
 from typing import Tuple
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
-from .run_tests_common import run_command
+from utils.utils import run_command
 
 _SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
 
@@ -53,7 +53,10 @@ def download_gcs_object_locally(gcsObjectUri: str) -> str:
  if returncode == 0:
  return fp.name
  else:
- raise f'failed to copy gcs object {gcsObjectUri} to local-file {fp.name}: returncode={returncode}. Deleting tempfile {fp.name}...'
+ raise Exception(
+ f'failed to copy gcs object {gcsObjectUri} to local-file {fp.name}:'
+ f' returncode={returncode}. Deleting tempfile {fp.name}...'
+ )
  os.remove(fp.name)
 
 

perfmetrics/scripts/testing_on_gke/examples/utils/gsheet_test.py

@@ -22,7 +22,7 @@
 from random import choices
 import string
 import unittest
-from gsheet import append_data_to_gsheet, download_gcs_object_as_tempfile, url
+from gsheet import append_data_to_gsheet, download_gcs_object_locally, url
 
 
 class GsheetTest(unittest.TestCase):
@@ -32,61 +32,48 @@ class GsheetTest(unittest.TestCase):
  # self.project_id = 'gcs-fuse-test'
 
  def test_append_data_to_gsheet(self):
- _DEFAULT_GSHEET_ID = '1UghIdsyarrV1HVNc6lugFZS1jJRumhdiWnPgoEC8Fe4'
+ _DEFAULT_GSHEET_ID = '1s9DCis6XZ_oHRIFTy0F8yVN93EGA2Koks_pzpCqAIS4'
 
- def _default_service_account_key_file(
- project_id: str, localfile: bool
- ) -> str:
- if localfile:
- if project_id == 'gcs-fuse-test':
- return '20240919-gcs-fuse-test-bc1a2c0aac45.json'
- elif project_id == 'gcs-fuse-test-ml':
- return '20240919-gcs-fuse-test-ml-d6e0247b2cf1.json'
- else:
- raise Exception(f'Unknown project-id: {project_id}')
+ def _default_service_account_key_file(project_id: str) -> str:
+ if project_id in ['gcs-fuse-test', 'gcs-fuse-test-ml']:
+ return f'gs://gcsfuse-aiml-test-outputs/creds/{project_id}.json'
  else:
- if project_id in ['gcs-fuse-test', 'gcs-fuse-test-ml']:
- return f'gs://gcsfuse-aiml-test-outputs/creds/{project_id}.json'
- else:
- raise Exception(f'Unknown project-id: {project_id}')
+ raise Exception(f'Unknown project-id: {project_id}')
 
  for project_id in ['gcs-fuse-test', 'gcs-fuse-test-ml']:
  for worksheet in ['fio-test', 'dlio-test']:
- for localkeyfile in [False]:
- serviceAccountKeyFile = _default_service_account_key_file(
- project_id, localkeyfile
- )
- append_data_to_gsheet(
- worksheet=worksheet,
- data={
- 'header': ('Column1', 'Column2'),
- 'values': [(
- ''.join(random.choices(string.ascii_letters, k=9)),
- random.random(),
- )],
- },
- serviceAccountKeyFile=serviceAccountKeyFile,
- gsheet_id=_DEFAULT_GSHEET_ID,
- )
+ serviceAccountKeyFile = _default_service_account_key_file(project_id)
+ append_data_to_gsheet(
+ worksheet=worksheet,
+ data={
+ 'header': ('Column1', 'Column2'),
+ 'values': [(
+ ''.join(random.choices(string.ascii_letters, k=9)),
+ random.random(),
+ )],
+ },
+ serviceAccountKeyFile=serviceAccountKeyFile,
+ gsheet_id=_DEFAULT_GSHEET_ID,
+ )
 
  def test_gsheet_url(self):
  gsheet_id = ''.join(random.choices(string.ascii_letters, k=20))
  gsheet_url = url(gsheet_id)
  self.assertTrue(gsheet_id in gsheet_url)
  self.assertTrue(len(gsheet_id) < len(gsheet_url))
 
- def test_download_gcs_object_as_tempfile(self):
+ def test_download_gcs_object_locally(self):
  gcs_object = 'gs://gcsfuse-aiml-test-outputs/creds/gcs-fuse-test.json'
- localfile = download_gcs_object_as_tempfile(gcs_object)
+ localfile = download_gcs_object_locally(gcs_object)
  self.assertTrue(localfile)
  self.assertTrue(localfile.strip())
  os.stat(localfile)
  os.remove(localfile)
 
- def test_download_gcs_object_as_tempfile_nonexistent(self):
+ def test_download_gcs_object_locally_nonexistent(self):
  gcs_object = 'gs://non/existing/gcs/object'
- localfile = download_gcs_object_as_tempfile(gcs_object)
- self.assertIsNone(localfile)
+ with self.assertRaises(Exception):
+  localfile = download_gcs_object_locally(gcs_object)
 
 
 if __name__ == '__main__':

perfmetrics/scripts/testing_on_gke/examples/utils/parse_logs_common.py

@@ -19,6 +19,7 @@
 import os
 import subprocess
 from typing import Tuple
+from utils.utils import run_command
 
 SUPPORTED_SCENARIOS = [
  "local-ssd",

perfmetrics/scripts/testing_on_gke/examples/utils/run_tests_common.py

@@ -18,21 +18,7 @@
 import argparse
 import subprocess
 import sys
-
-
-def run_command(command: str) -> int:
- """Runs the given string command as a subprocess.
-
- Returns exit-code which would be non-zero for error.
- """
- result = subprocess.run(
- [word for word in command.split(' ') if (word and not str.isspace(word))],
- capture_output=True,
- text=True,
- )
- print(result.stdout)
- print(result.stderr)
- return result.returncode
+from utils import run_command
 
 
 def escape_commas_in_string(unescapedStr: str) -> str:

perfmetrics/scripts/testing_on_gke/examples/utils/utils.py

@@ -359,3 +359,18 @@ def get_cpu_from_monitoring_api(
  ),
  5, # round up to 5 decimal places.
  )
+
+
+def run_command(command: str) -> int:
+ """Runs the given string command as a subprocess.
+
+ Returns exit-code which would be non-zero for error.
+ """
+ result = subprocess.run(
+ [word for word in command.split(" ") if (word and not str.isspace(word))],
+ capture_output=True,
+ text=True,
+ )
+ print(result.stdout)
+ print(result.stderr)
+ return result.returncode