From 457f2f4d4e5c44329a5006a0a44cb408750d4b06 Mon Sep 17 00:00:00 2001 From: uiro-bi Date: Wed, 11 Sep 2024 09:39:25 -0300 Subject: [PATCH 1/3] fix br_ibge_inpc.mes_categoria_municipio schedule --- pipelines/datasets/br_ibge_inpc/schedules.py | 2 +- pipelines/utils/crawler_ibge_inflacao/flows.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/datasets/br_ibge_inpc/schedules.py b/pipelines/datasets/br_ibge_inpc/schedules.py index 9b6802e21..b0466b341 100644 --- a/pipelines/datasets/br_ibge_inpc/schedules.py +++ b/pipelines/datasets/br_ibge_inpc/schedules.py @@ -65,7 +65,7 @@ schedule_br_ibge_inpc_mes_categoria_municipio = Schedule( clocks=[ CronClock( - cron="30 14 * * *", # everyday at 14:00:00 + cron="30 14 * * *", # everyday at 14:30:00 start_date=datetime(2023, 10, 6, 0, 0), labels=[ constants.BASEDOSDADOS_PROD_AGENT_LABEL.value, diff --git a/pipelines/utils/crawler_ibge_inflacao/flows.py b/pipelines/utils/crawler_ibge_inflacao/flows.py index 1a9ec296e..a9da59761 100644 --- a/pipelines/utils/crawler_ibge_inflacao/flows.py +++ b/pipelines/utils/crawler_ibge_inflacao/flows.py @@ -317,7 +317,7 @@ indice=INDICE, dataset_id=dataset_id, table_id=table_id ) - with case(needs_to_update[0], True): + with case(needs_to_update[0], False): was_downloaded = crawler( indice=INDICE, folder=FOLDER, upstream_tasks=[needs_to_update] ) From 5f17cfa419e5eb8d4381eb1cdab22fbdb634526a Mon Sep 17 00:00:00 2001 From: uiro-bi Date: Wed, 11 Sep 2024 09:46:29 -0300 Subject: [PATCH 2/3] revert test changes --- pipelines/utils/crawler_ibge_inflacao/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/utils/crawler_ibge_inflacao/flows.py b/pipelines/utils/crawler_ibge_inflacao/flows.py index a9da59761..1a9ec296e 100644 --- a/pipelines/utils/crawler_ibge_inflacao/flows.py +++ b/pipelines/utils/crawler_ibge_inflacao/flows.py @@ -317,7 +317,7 @@ indice=INDICE, dataset_id=dataset_id, table_id=table_id ) - with case(needs_to_update[0], False): + with case(needs_to_update[0], True): was_downloaded = crawler( indice=INDICE, folder=FOLDER, upstream_tasks=[needs_to_update] ) From 2c627ba80719274667ca992668d916b85b277161 Mon Sep 17 00:00:00 2001 From: uiro-bi Date: Thu, 12 Sep 2024 10:34:27 -0300 Subject: [PATCH 3/3] feat: update code owners --- pipelines/datasets/br_ibge_inpc/flows.py | 8 ++++---- pipelines/datasets/br_ibge_ipca/flows.py | 8 ++++---- pipelines/datasets/br_ibge_ipca15/flows.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pipelines/datasets/br_ibge_inpc/flows.py b/pipelines/datasets/br_ibge_inpc/flows.py index 11082b761..43128cd17 100644 --- a/pipelines/datasets/br_ibge_inpc/flows.py +++ b/pipelines/datasets/br_ibge_inpc/flows.py @@ -29,7 +29,7 @@ br_ibge_inpc_mes_categoria_brasil = deepcopy(flow_ibge_inflacao_mes_brasil) br_ibge_inpc_mes_categoria_brasil.name = "br_ibge_inpc.mes_categoria_brasil" -br_ibge_inpc_mes_categoria_brasil.code_owners = ["equipe_pipelines"] +br_ibge_inpc_mes_categoria_brasil.code_owners = ["Gabriel Pisa"] br_ibge_inpc_mes_categoria_brasil.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_inpc_mes_categoria_brasil.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -39,7 +39,7 @@ br_ibge_inpc_mes_categoria_rm = deepcopy(flow_ibge_inflacao_mes_rm) br_ibge_inpc_mes_categoria_rm.name = "br_ibge_inpc.mes_categoria_rm" -br_ibge_inpc_mes_categoria_rm.code_owners = ["equipe_pipelines"] +br_ibge_inpc_mes_categoria_rm.code_owners = ["Gabriel Pisa"] br_ibge_inpc_mes_categoria_rm.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_inpc_mes_categoria_rm.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -48,7 +48,7 @@ br_ibge_inpc_mes_categoria_municipio = deepcopy(flow_ibge_inflacao_mes_municipio) br_ibge_inpc_mes_categoria_municipio.name = "br_ibge_inpc.mes_categoria_municipio" -br_ibge_inpc_mes_categoria_municipio.code_owners = ["equipe_pipelines"] +br_ibge_inpc_mes_categoria_municipio.code_owners = ["Gabriel Pisa"] br_ibge_inpc_mes_categoria_municipio.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_inpc_mes_categoria_municipio.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -59,7 +59,7 @@ br_ibge_inpc_mes_brasil = deepcopy(flow_ibge_inflacao_mes_geral) br_ibge_inpc_mes_brasil.name = "br_ibge_inpc.mes_brasil" -br_ibge_inpc_mes_brasil.code_owners = ["equipe_pipelines"] +br_ibge_inpc_mes_brasil.code_owners = ["Gabriel Pisa"] br_ibge_inpc_mes_brasil.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_inpc_mes_brasil.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value) br_ibge_inpc_mes_brasil.schedule = schedule_br_ibge_inpc_mes_brasil diff --git a/pipelines/datasets/br_ibge_ipca/flows.py b/pipelines/datasets/br_ibge_ipca/flows.py index 57c5aeb7a..f3fe247d5 100644 --- a/pipelines/datasets/br_ibge_ipca/flows.py +++ b/pipelines/datasets/br_ibge_ipca/flows.py @@ -25,7 +25,7 @@ br_ibge_ipca_mes_categoria_brasil = deepcopy(flow_ibge_inflacao_mes_brasil) br_ibge_ipca_mes_categoria_brasil.name = "br_ibge_ipca.mes_categoria_brasil" -br_ibge_ipca_mes_categoria_brasil.code_owners = ["equipe_pipelines"] +br_ibge_ipca_mes_categoria_brasil.code_owners = ["Gabriel Pisa"] br_ibge_ipca_mes_categoria_brasil.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca_mes_categoria_brasil.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -35,7 +35,7 @@ br_ibge_ipca_mes_categoria_rm = deepcopy(flow_ibge_inflacao_mes_rm) br_ibge_ipca_mes_categoria_rm.name = "br_ibge_ipca.mes_categoria_rm" -br_ibge_ipca_mes_categoria_rm.code_owners = ["equipe_pipelines"] +br_ibge_ipca_mes_categoria_rm.code_owners = ["Gabriel Pisa"] br_ibge_ipca_mes_categoria_rm.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca_mes_categoria_rm.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -44,7 +44,7 @@ br_ibge_ipca_mes_categoria_municipio = deepcopy(flow_ibge_inflacao_mes_municipio) br_ibge_ipca_mes_categoria_municipio.name = "br_ibge_ipca.mes_categoria_municipio" -br_ibge_ipca_mes_categoria_municipio.code_owners = ["equipe_pipelines"] +br_ibge_ipca_mes_categoria_municipio.code_owners = ["Gabriel Pisa"] br_ibge_ipca_mes_categoria_municipio.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca_mes_categoria_municipio.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -56,7 +56,7 @@ br_ibge_ipca_mes_brasil = deepcopy(flow_ibge_inflacao_mes_geral) br_ibge_ipca_mes_brasil.name = "br_ibge_ipca.mes_brasil" -br_ibge_ipca_mes_brasil.code_owners = ["equipe_pipelines"] +br_ibge_ipca_mes_brasil.code_owners = ["Gabriel Pisa"] br_ibge_ipca_mes_brasil.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca_mes_brasil.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value) br_ibge_ipca_mes_brasil.schedule = schedule_br_ibge_ipca_mes_brasil diff --git a/pipelines/datasets/br_ibge_ipca15/flows.py b/pipelines/datasets/br_ibge_ipca15/flows.py index 557dce2a3..fc513e583 100644 --- a/pipelines/datasets/br_ibge_ipca15/flows.py +++ b/pipelines/datasets/br_ibge_ipca15/flows.py @@ -29,7 +29,7 @@ br_ibge_ipca15_mes_categoria_brasil = deepcopy(flow_ibge_inflacao_mes_brasil) br_ibge_ipca15_mes_categoria_brasil.name = "br_ibge_ipca15.mes_categoria_brasil" -br_ibge_ipca15_mes_categoria_brasil.code_owners = ["equipe_pipelines"] +br_ibge_ipca15_mes_categoria_brasil.code_owners = ["Gabriel Pisa"] br_ibge_ipca15_mes_categoria_brasil.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca15_mes_categoria_brasil.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -41,7 +41,7 @@ br_ibge_ipca15_mes_categoria_rm = deepcopy(flow_ibge_inflacao_mes_rm) br_ibge_ipca15_mes_categoria_rm.name = "br_ibge_ipca15.mes_categoria_rm" -br_ibge_ipca15_mes_categoria_rm.code_owners = ["equipe_pipelines"] +br_ibge_ipca15_mes_categoria_rm.code_owners = ["Gabriel Pisa"] br_ibge_ipca15_mes_categoria_rm.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca15_mes_categoria_rm.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -50,7 +50,7 @@ br_ibge_ipca15_mes_categoria_municipio = deepcopy(flow_ibge_inflacao_mes_municipio) br_ibge_ipca15_mes_categoria_municipio.name = "br_ibge_ipca15.mes_categoria_municipio" -br_ibge_ipca15_mes_categoria_municipio.code_owners = ["equipe_pipelines"] +br_ibge_ipca15_mes_categoria_municipio.code_owners = ["Gabriel Pisa"] br_ibge_ipca15_mes_categoria_municipio.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca15_mes_categoria_municipio.run_config = KubernetesRun( image=constants.DOCKER_IMAGE.value @@ -61,7 +61,7 @@ br_ibge_ipca15_mes_brasil = deepcopy(flow_ibge_inflacao_mes_geral) br_ibge_ipca15_mes_brasil.name = "br_ibge_ipca15.mes_brasil" -br_ibge_ipca15_mes_brasil.code_owners = ["equipe_pipelines"] +br_ibge_ipca15_mes_brasil.code_owners = ["Gabriel Pisa"] br_ibge_ipca15_mes_brasil.storage = GCS(constants.GCS_FLOWS_BUCKET.value) br_ibge_ipca15_mes_brasil.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value) br_ibge_ipca15_mes_brasil.schedule = schedule_br_ibge_ipca15_mes_brasil