Merge pull request #355 from basedosdados/staging/update_metadata

[feat] update_metadata
basedosdados · Jul 18, 2023 · f531f31 · f531f31
2 parents f83beb1 + 0e7a004
commit f531f31
Show file tree

Hide file tree

Showing 2 changed files with 186 additions and 115 deletions.
diff --git a/pipelines/utils/tasks.py b/pipelines/utils/tasks.py
@@ -19,14 +19,15 @@
 from pipelines.constants import constants
 from pipelines.utils.utils import (
  dump_header_to_csv,
- find_ids,
+ get_ids,
  parse_temporal_coverage,
  get_credentials_utils,
  create_update,
  extract_last_update,
  get_first_date,
  log,
  get_credentials_from_secret,
+ get_token,
 )
 from typing import Tuple
 
@@ -371,7 +372,9 @@ def update_django_metadata(
  table_id: str,
  metadata_type: str,
  _last_date=None,
+ date_format: str = "yy-mm-dd",
  bq_last_update: bool = True,
+ api_mode: str = "prod",
 ):
  """
  Updates Django metadata.
@@ -381,8 +384,10 @@ def update_django_metadata(
  table_id (str): The ID of the table.
  metadata_type (str): The type of metadata to update.
  _last_date (optional): The last date for metadata update if `bq_last_update` is False. Defaults to None.
+ date_format (str, optional): The date format to use when parsing dates ('yy-mm-dd', 'yy-mm', or 'yy'). Defaults to 'yy-mm-dd'.
  bq_last_update (bool, optional): Flag indicating whether to use BigQuery's last update date for metadata.
  If True, `_last_date` is ignored. Defaults to True.
+ api_mode (str, optional): The API mode to be used ('prod', 'staging'). Defaults to 'prod'.
 
  Returns:
  None
@@ -391,30 +396,25 @@ def update_django_metadata(
  Exception: If the metadata_type is not supported.
 
  """
- (email, password) = get_credentials_utils(secret_path="api_user_prod")
+ (email, password) = get_credentials_utils(secret_path=f"api_user_{api_mode}")
 
- ids = find_ids(
+ ids = get_ids(
  dataset_id,
  table_id,
  email,
  password,
  )
+ log(f"IDS:{ids}")
 
  if metadata_type == "DateTimeRange":
  if bq_last_update:
  last_date = extract_last_update(
  dataset_id,
  table_id,
- )
- first_date = get_first_date(
- ids,
- email,
- password,
+ date_format,
  )
 
- resource_to_temporal_coverage = parse_temporal_coverage(
- f"{first_date}{last_date}"
- )
+ resource_to_temporal_coverage = parse_temporal_coverage(f"{last_date}")
  resource_to_temporal_coverage["coverage"] = ids.get("coverage_id")
  log(f"Mutation parameters: {resource_to_temporal_coverage}")
 
@@ -426,19 +426,13 @@ def update_django_metadata(
  update=True,
  email=email,
  password=password,
+ api_mode=api_mode,
  )
  else:
  last_date = _last_date
  log(f"Última data {last_date}")
- first_date = get_first_date(
- ids,
- email,
- password,
- )
 
- resource_to_temporal_coverage = parse_temporal_coverage(
- f"{first_date}{last_date}"
- )
+ resource_to_temporal_coverage = parse_temporal_coverage(f"{last_date}")
 
  resource_to_temporal_coverage["coverage"] = ids.get("coverage_id")
  log(f"Mutation parameters: {resource_to_temporal_coverage}")
@@ -451,4 +445,5 @@ def update_django_metadata(
  update=True,
  email=email,
  password=password,
+ api_mode=api_mode,
  )