From c0b57a5d30fd9a654e41d52d0114c91660a6f7b7 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 9 May 2024 16:46:11 -0300 Subject: [PATCH 1/9] feat: add pericia and folha dataset --- .../dump_db_ergon_folha_pagamento/flows.py | 50 +++++++++++++++++ .../schedules.py | 55 +++++++++++++++++++ .../dump_db_ergon_pericia_medica/flows.py | 48 ++++++++++++++++ .../dump_db_ergon_pericia_medica/schedules.py | 55 +++++++++++++++++++ queries/dbt_project.yml | 10 +++- .../folha_pagamento.sql | 2 + .../pericia_medica.sql | 2 + 7 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py create mode 100644 pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py create mode 100644 pipelines/ergon/dump_db_ergon_pericia_medica/flows.py create mode 100644 pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py create mode 100644 queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql create mode 100644 queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql diff --git a/pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py b/pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py new file mode 100644 index 0000000..d43ae23 --- /dev/null +++ b/pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +Database dumping flows for segovi project. +""" + +from copy import deepcopy + +from prefect.run_configs import KubernetesRun +from prefect.storage import GCS +from prefeitura_rio.pipelines_templates.dump_db.flows import flow as dump_sql_flow +from prefeitura_rio.pipelines_utils.prefect import set_default_parameters +from prefeitura_rio.pipelines_utils.state_handlers import ( + handler_initialize_sentry, + handler_inject_bd_credentials, +) + +from pipelines.constants import constants +from pipelines.ergon.dump_db_ergon_folha_pagamento.schedules import ( + ergon_folha_pagamento_monthly_update_schedule, +) + +dump_db_ergon_folha_pagamento_flow = deepcopy(dump_sql_flow) +dump_db_ergon_folha_pagamento_flow.state_handlers = [ + handler_inject_bd_credentials, + handler_initialize_sentry, +] +dump_db_ergon_folha_pagamento_flow.name = ( + "SMFP: ergon folha pagamento - Ingerir tabelas de banco SQL" +) +dump_db_ergon_folha_pagamento_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value) +dump_db_ergon_folha_pagamento_flow.run_config = KubernetesRun( + image=constants.DOCKER_IMAGE.value, + labels=[ + constants.RJ_SMFP_AGENT_LABEL.value, + ], +) + +ergon_folha_pagamento_default_parameters = { + "db_database": "P01.PCRJ", + "db_host": "10.70.6.21", + "db_port": "1526", + "db_type": "oracle", + "infisical_secret_path": "/db-ergon-prod", + "dataset_id": "recursos_humanos_ergon_folha_pagamento", +} +dump_db_ergon_folha_pagamento_flow = set_default_parameters( + dump_db_ergon_folha_pagamento_flow, default_parameters=ergon_folha_pagamento_default_parameters +) + +dump_db_ergon_folha_pagamento_flow.schedule = ergon_folha_pagamento_monthly_update_schedule diff --git a/pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py b/pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py new file mode 100644 index 0000000..88880ea --- /dev/null +++ b/pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# flake8: noqa: E501 +""" +Schedules for the database dump pipeline +""" + +from datetime import datetime, timedelta + +import pytz +from prefect.schedules import Schedule +from prefeitura_rio.pipelines_utils.io import untuple_clocks as untuple +from prefeitura_rio.pipelines_utils.prefect import generate_dump_db_schedules + +from pipelines.constants import constants + +##################################### +# +# Ergon Schedules +# +##################################### + +ergon_queries = { + "folha_pagamento": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + CARGO, NOME, CATEGORIA, SUBCATEGORIA, DT_EXTINCAO, + CONTROLE_VAGA, POR_REFER, E_AGLUTINADOR, CARGO_AGLUT, + TIPO_CARGO, CARGO_FUNCAO, ESCOLARIDADE, FLEX_CAMPO_09, + PONTPUBL, DT_CONTROLE_ACUM, PONTLEI, DT_INICIO_CONTR_VAGA, + ID_REG + FROM C_ERGON.VW_DLK_ERG_CARGOS_ + """, + }, +} + +ergon_clocks = generate_dump_db_schedules( + interval=timedelta(days=1), + start_date=datetime(2022, 11, 9, 22, 30, tzinfo=pytz.timezone("America/Sao_Paulo")), + labels=[ + constants.RJ_SMFP_AGENT_LABEL.value, + ], + db_database="P01.PCRJ", + db_host="10.70.6.21", + db_port="1526", + db_type="oracle", + dataset_id="recursos_humanos_ergon_folha_pagamento", + infisical_secret_path="/db-ergon-prod", + table_parameters=ergon_queries, +) + +ergon_folha_pagamento_monthly_update_schedule = Schedule(clocks=untuple(ergon_clocks)) diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py b/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py new file mode 100644 index 0000000..0795e3e --- /dev/null +++ b/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +Database dumping flows for segovi project. +""" + +from copy import deepcopy + +from prefect.run_configs import KubernetesRun +from prefect.storage import GCS +from prefeitura_rio.pipelines_templates.dump_db.flows import flow as dump_sql_flow +from prefeitura_rio.pipelines_utils.prefect import set_default_parameters +from prefeitura_rio.pipelines_utils.state_handlers import ( + handler_initialize_sentry, + handler_inject_bd_credentials, +) + +from pipelines.constants import constants +from pipelines.ergon.dump_db_ergon_pericia_medica.schedules import ( + ergon_pericia_medica_monthly_update_schedule, +) + +dump_db_ergon_pericia_medica_flow = deepcopy(dump_sql_flow) +dump_db_ergon_pericia_medica_flow.state_handlers = [ + handler_inject_bd_credentials, + handler_initialize_sentry, +] +dump_db_ergon_pericia_medica_flow.name = "SMFP: ergon pericia medica - Ingerir tabelas de banco SQL" +dump_db_ergon_pericia_medica_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value) +dump_db_ergon_pericia_medica_flow.run_config = KubernetesRun( + image=constants.DOCKER_IMAGE.value, + labels=[ + constants.RJ_SMFP_AGENT_LABEL.value, + ], +) + +ergon_pericia_medica_default_parameters = { + "db_database": "P01.PCRJ", + "db_host": "10.70.6.21", + "db_port": "1526", + "db_type": "oracle", + "infisical_secret_path": "/db-ergon-prod", + "dataset_id": "recursos_humanos_ergon_pericia_medica", +} +dump_db_ergon_pericia_medica_flow = set_default_parameters( + dump_db_ergon_pericia_medica_flow, default_parameters=ergon_pericia_medica_default_parameters +) + +dump_db_ergon_pericia_medica_flow.schedule = ergon_pericia_medica_monthly_update_schedule diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py new file mode 100644 index 0000000..6a50732 --- /dev/null +++ b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# flake8: noqa: E501 +""" +Schedules for the database dump pipeline +""" + +from datetime import datetime, timedelta + +import pytz +from prefect.schedules import Schedule +from prefeitura_rio.pipelines_utils.io import untuple_clocks as untuple +from prefeitura_rio.pipelines_utils.prefect import generate_dump_db_schedules + +from pipelines.constants import constants + +##################################### +# +# Ergon Schedules +# +##################################### + +ergon_queries = { + "folha_pagamento": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + CARGO, NOME, CATEGORIA, SUBCATEGORIA, DT_EXTINCAO, + CONTROLE_VAGA, POR_REFER, E_AGLUTINADOR, CARGO_AGLUT, + TIPO_CARGO, CARGO_FUNCAO, ESCOLARIDADE, FLEX_CAMPO_09, + PONTPUBL, DT_CONTROLE_ACUM, PONTLEI, DT_INICIO_CONTR_VAGA, + ID_REG + FROM C_ERGON.VW_DLK_ERG_CARGOS_ + """, + }, +} + +ergon_clocks = generate_dump_db_schedules( + interval=timedelta(days=1), + start_date=datetime(2022, 11, 9, 22, 30, tzinfo=pytz.timezone("America/Sao_Paulo")), + labels=[ + constants.RJ_SMFP_AGENT_LABEL.value, + ], + db_database="P01.PCRJ", + db_host="10.70.6.21", + db_port="1526", + db_type="oracle", + dataset_id="recursos_humanos_ergon_pericia_medica", + infisical_secret_path="/db-ergon-prod", + table_parameters=ergon_queries, +) + +ergon_pericia_medica_monthly_update_schedule = Schedule(clocks=untuple(ergon_clocks)) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index d24d2d1..d3c4517 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -47,4 +47,12 @@ models: +schema: adm_instrumentos_firmados compras_materiais_servicos_sigma: +materialized: table - +schema: compras_materiais_servicos_sigma \ No newline at end of file + +schema: compras_materiais_servicos_sigma + recursos_humanos_ergon_folha_pagamento: + +materialized: table + +schema: recursos_humanos_ergon_folha_pagamento + recursos_humanos_ergon_pericia_medica: + +materialized: table + +schema: recursos_humanos_ergon_pericia_medica + + diff --git a/queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql b/queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql new file mode 100644 index 0000000..7f077bb --- /dev/null +++ b/queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql @@ -0,0 +1,2 @@ +SELECT * +FROM rj-smfp.recursos_humanos_ergon_folha_pagamento_staging.folha_pagamento AS t \ No newline at end of file diff --git a/queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql b/queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql new file mode 100644 index 0000000..a77fb57 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql @@ -0,0 +1,2 @@ +SELECT * +FROM rj-smfp.recursos_humanos_ergon_folha_pagamento_staging.pericia_medica AS t \ No newline at end of file From c8d341dfd18fdc9cea4cc53a4113412400ef7c00 Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 10 May 2024 13:32:40 -0300 Subject: [PATCH 2/9] feat: add new dataset --- Dockerfile | 38 +-- pipelines/ergon/dump_db_ergon/schedules.py | 108 ++----- .../dump_db_ergon_folha_pagamento/flows.py | 50 ---- .../schedules.py | 55 ---- .../dump_db_ergon_pericia_medica/schedules.py | 273 +++++++++++++++++- .../models/recursos_humanos_ergon/rubrica.sql | 2 + .../folha_pagamento.sql | 2 - .../pericia_medica.sql | 2 - .../cid.sql | 3 + .../designacoes_ev.sql | 3 + .../erg_pm_cid10_capitulo.sql | 3 + .../erg_pm_decisao.sql | 3 + .../erg_pm_grupoexame.sql | 3 + .../erg_pm_pront_med.sql | 3 + .../erg_pm_resultpront.sql | 3 + .../exames.sql | 3 + .../funcoes_ev.sql | 3 + .../medicos.sql | 3 + .../prontuario.sql | 3 + .../tpmrj_pm_juntas.sql | 3 + .../tpmrj_pm_periciasjunta.sql | 3 + .../tpmrj_pm_resultjunta.sql | 3 + .../tpmrj_pm_vwbim .sql | 3 + .../tpmrj_pm_vwtpconclper .sql | 3 + 24 files changed, 348 insertions(+), 230 deletions(-) delete mode 100644 pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py delete mode 100644 pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py create mode 100644 queries/models/recursos_humanos_ergon/rubrica.sql delete mode 100644 queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql delete mode 100644 queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/cid.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/designacoes_ev.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_cid10_capitulo.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_decisao.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_grupoexame.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_pront_med.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_resultpront.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/exames.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/funcoes_ev.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/medicos.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/prontuario.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_juntas.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_periciasjunta.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_resultjunta.sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim .sql create mode 100644 queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper .sql diff --git a/Dockerfile b/Dockerfile index 5c02124..e79e4a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,25 @@ # Build arguments -ARG PYTHON_VERSION=3.10-slim - -# Get Oracle Instant Client -FROM curlimages/curl:7.81.0 as curl-step -ARG ORACLE_INSTANT_CLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/215000/instantclient-basic-linux.x64-21.5.0.0.0dbru.zip -RUN curl -sSLo /tmp/instantclient.zip $ORACLE_INSTANT_CLIENT_URL - -# Unzip Oracle Instant Client -FROM ubuntu:18.04 as unzip-step -COPY --from=curl-step /tmp/instantclient.zip /tmp/instantclient.zip -RUN apt-get update && \ - apt-get install --no-install-recommends -y unzip && \ - rm -rf /var/lib/apt/lists/* && \ - unzip /tmp/instantclient.zip -d /tmp +ARG PYTHON_VERSION=3.10-slim-buster # Start Python image FROM python:${PYTHON_VERSION} -# Install git + +# Install a few dependencies RUN apt-get update && \ - apt-get install -y git && \ + apt-get install --no-install-recommends -y git curl gnupg2 libaio1 && \ + curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ + echo "deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list && \ + apt-get update && \ + ACCEPT_EULA=Y apt-get install -y git msodbcsql17 openssl unixodbc-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +COPY ./openssl.cnf /etc/ssl/openssl.cnf # Setting environment with prefect version ARG PREFECT_VERSION=1.4.1 ENV PREFECT_VERSION $PREFECT_VERSION -# Setup Oracle Instant Client and SQL Server ODBC Driver -WORKDIR /opt/oracle -COPY --from=unzip-step /tmp/instantclient_21_5 /opt/oracle/instantclient_21_5 -RUN apt-get update && \ - apt-get install --no-install-recommends -y curl gnupg2 libaio1 && \ - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ - echo "deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list && \ - apt-get update && \ - ACCEPT_EULA=Y apt-get install --no-install-recommends -y ffmpeg libsm6 libxext6 msodbcsql17 openssl unixodbc-dev && \ - rm -rf /var/lib/apt/lists/* && \ - sh -c "echo /opt/oracle/instantclient_21_5 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ - ldconfig - # Setup virtual environment and prefect ENV VIRTUAL_ENV=/opt/venv RUN python3 -m venv $VIRTUAL_ENV diff --git a/pipelines/ergon/dump_db_ergon/schedules.py b/pipelines/ergon/dump_db_ergon/schedules.py index c8db273..f5fba17 100644 --- a/pipelines/ergon/dump_db_ergon/schedules.py +++ b/pipelines/ergon/dump_db_ergon/schedules.py @@ -484,91 +484,29 @@ FROM ERGON.IPL_PT_FICHAS """, }, - # "rubrica": { - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, - # "folha_empresa_original": { - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, - # "tpmrj_pm_vwsolicbim": { - # "dataset_id": "recursos_humanos_ergon_bim", - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, - # "tpmrj_pm_vwbim": { - # "dataset_id": "recursos_humanos_ergon_bim", - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, - # "tpmrj_pm_solic_bim": { - # "dataset_id": "recursos_humanos_ergon_bim", - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, - # "tpmrj_pm_bim": { - # "dataset_id": "recursos_humanos_ergon_bim", - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, - # "tpmrj_pm_solic_bim_vinc": { - # "dataset_id": "recursos_humanos_ergon_bim", - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, - # "tpmrj_pm_vwafastamentoeferias": { - # "materialize_after_dump": True, - # "biglake_table": True, - # "materialization_mode": "prod", - # "dump_mode": "append", - # "lower_bound_date": "current_month", - # "partition_columns": "MES_ANO_FOLHA", - # "execute_query": """ - # """, - # }, + "rubrica": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + RUBRICA, NOME, MNEMONICO, NOME_ABREV, + TIPORUBR, FAT_VANT, FAT_IR, FAT_PREV, + FAT_IRFER, FAT_PREVFER, FAT_RAIS, E_CONS, + E_PA, E_SALFAM, E_IR, E_PREV, COMISSAO, + FLEX_CAMPO_01, FLEX_CAMPO_02, FLEX_CAMPO_03, + FLEX_CAMPO_04, FLEX_CAMPO_05, RUB, USA_COMPLEMENTO, + FORMULA_PADRAO_02, FORMULA_PADRAO_12, PONTLEI, SQL_LOVCOMPL, + VALIDA_COMPL_BD, FORMATO_COMPLEMENTO, FORMULA_PADRAO_02PER, + FORMULA_PADRAO_12PER, MODALIDADE_CALCULO, FLEX_CAMPO_06, + FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10, + PONTPUBL, FAT_VANT_REPASSE, E_REPASSE, FLEX_CAMPO_11, FLEX_CAMPO_12, + FLEX_CAMPO_13, FLEX_CAMPO_14, FLEX_CAMPO_15, FLEX_CAMPO_16, FLEX_CAMPO_17, + FLEX_CAMPO_18, FLEX_CAMPO_19, FLEX_CAMPO_20, ID_REG + FROM ERGON.RUBRICAS + """, + }, } ergon_clocks = generate_dump_db_schedules( diff --git a/pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py b/pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py deleted file mode 100644 index d43ae23..0000000 --- a/pipelines/ergon/dump_db_ergon_folha_pagamento/flows.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Database dumping flows for segovi project. -""" - -from copy import deepcopy - -from prefect.run_configs import KubernetesRun -from prefect.storage import GCS -from prefeitura_rio.pipelines_templates.dump_db.flows import flow as dump_sql_flow -from prefeitura_rio.pipelines_utils.prefect import set_default_parameters -from prefeitura_rio.pipelines_utils.state_handlers import ( - handler_initialize_sentry, - handler_inject_bd_credentials, -) - -from pipelines.constants import constants -from pipelines.ergon.dump_db_ergon_folha_pagamento.schedules import ( - ergon_folha_pagamento_monthly_update_schedule, -) - -dump_db_ergon_folha_pagamento_flow = deepcopy(dump_sql_flow) -dump_db_ergon_folha_pagamento_flow.state_handlers = [ - handler_inject_bd_credentials, - handler_initialize_sentry, -] -dump_db_ergon_folha_pagamento_flow.name = ( - "SMFP: ergon folha pagamento - Ingerir tabelas de banco SQL" -) -dump_db_ergon_folha_pagamento_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value) -dump_db_ergon_folha_pagamento_flow.run_config = KubernetesRun( - image=constants.DOCKER_IMAGE.value, - labels=[ - constants.RJ_SMFP_AGENT_LABEL.value, - ], -) - -ergon_folha_pagamento_default_parameters = { - "db_database": "P01.PCRJ", - "db_host": "10.70.6.21", - "db_port": "1526", - "db_type": "oracle", - "infisical_secret_path": "/db-ergon-prod", - "dataset_id": "recursos_humanos_ergon_folha_pagamento", -} -dump_db_ergon_folha_pagamento_flow = set_default_parameters( - dump_db_ergon_folha_pagamento_flow, default_parameters=ergon_folha_pagamento_default_parameters -) - -dump_db_ergon_folha_pagamento_flow.schedule = ergon_folha_pagamento_monthly_update_schedule diff --git a/pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py b/pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py deleted file mode 100644 index 88880ea..0000000 --- a/pipelines/ergon/dump_db_ergon_folha_pagamento/schedules.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa: E501 -""" -Schedules for the database dump pipeline -""" - -from datetime import datetime, timedelta - -import pytz -from prefect.schedules import Schedule -from prefeitura_rio.pipelines_utils.io import untuple_clocks as untuple -from prefeitura_rio.pipelines_utils.prefect import generate_dump_db_schedules - -from pipelines.constants import constants - -##################################### -# -# Ergon Schedules -# -##################################### - -ergon_queries = { - "folha_pagamento": { - "materialize_after_dump": True, - "biglake_table": True, - "materialization_mode": "prod", - "dump_mode": "overwrite", - "execute_query": """ - SELECT - CARGO, NOME, CATEGORIA, SUBCATEGORIA, DT_EXTINCAO, - CONTROLE_VAGA, POR_REFER, E_AGLUTINADOR, CARGO_AGLUT, - TIPO_CARGO, CARGO_FUNCAO, ESCOLARIDADE, FLEX_CAMPO_09, - PONTPUBL, DT_CONTROLE_ACUM, PONTLEI, DT_INICIO_CONTR_VAGA, - ID_REG - FROM C_ERGON.VW_DLK_ERG_CARGOS_ - """, - }, -} - -ergon_clocks = generate_dump_db_schedules( - interval=timedelta(days=1), - start_date=datetime(2022, 11, 9, 22, 30, tzinfo=pytz.timezone("America/Sao_Paulo")), - labels=[ - constants.RJ_SMFP_AGENT_LABEL.value, - ], - db_database="P01.PCRJ", - db_host="10.70.6.21", - db_port="1526", - db_type="oracle", - dataset_id="recursos_humanos_ergon_folha_pagamento", - infisical_secret_path="/db-ergon-prod", - table_parameters=ergon_queries, -) - -ergon_folha_pagamento_monthly_update_schedule = Schedule(clocks=untuple(ergon_clocks)) diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py index 6a50732..3492da1 100644 --- a/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py +++ b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py @@ -20,23 +20,272 @@ ##################################### ergon_queries = { - "folha_pagamento": { + "cid": { "materialize_after_dump": True, "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ SELECT - CARGO, NOME, CATEGORIA, SUBCATEGORIA, DT_EXTINCAO, - CONTROLE_VAGA, POR_REFER, E_AGLUTINADOR, CARGO_AGLUT, - TIPO_CARGO, CARGO_FUNCAO, ESCOLARIDADE, FLEX_CAMPO_09, - PONTPUBL, DT_CONTROLE_ACUM, PONTLEI, DT_INICIO_CONTR_VAGA, - ID_REG - FROM C_ERGON.VW_DLK_ERG_CARGOS_ + TIPO, CODIGO, DESCRICAO, COD_PAI, FLEX_CAMPO_01, + FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05, + FLEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, + FLEX_CAMPO_10, FLEX_CAMPO_11, FLEX_CAMPO_12, FLEX_CAMPO_13, + FLEX_CAMPO_14, FLEX_CAMPO_15, CAPITULO, GUID_GRUPO, + DESCRICAO_ABREV, CLASSIF, REFER, RESTR_SEXO, CAUSA_OBITO, + EXCLUIDOS, NIVEL + FROM ERGON.CID + """, + }, + "designacoes_ev": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + NUMFUNC, NUMVINC, DTINI, DTFIM, FUNCAO, SETOR, OBS, + NUMFUNCSUBS1, NUMVINCSUBS1, NUMFUNCSUBS2, NUMVINCSUBS2, + PONTPUBL, NUMERO_VAGA, PONTLEI, FLEX_CAMPO_01, FLEX_CAMPO_02, + FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05, EMP_CODIGO, ID_REG + FROM ERGON.DESIGNACOES_EV + """, + }, + "erg_pm_cid10_capitulo": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + CAPITULO, DESCRICAO, FLEX_CAMPO_01, + FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05 + FROM ERGON.ERG_PM_CID10_CAPITULO + """, + }, + "erg_pm_decisao": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + DECISAO, SIGLAEXAME, SIGLA, DESCRICAO, GERA_LICAFAST, + GERA_READAPTACAO, ALTA_AT, NEGADO, QUANT_PERITOS, TEXTO_LAUDO, + NUM_MESES, RETIFICA, INTERVALO_DIAS, FLEX_CAMPO_01, + FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05, + FLEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, + FLEX_CAMPO_10, CONCLUI_INGRESSO, FLEX_CAMPO_11, FLEX_CAMPO_12, + FLEX_CAMPO_13, FLEX_CAMPO_14, FLEX_CAMPO_15, FLEX_CAMPO_16, FLEX_CAMPO_17, + FLEX_CAMPO_18, FLEX_CAMPO_19, FLEX_CAMPO_20, EXAME_ENCAM, RETIF_POR_EXCLUSAO, + TITULO_LAUDO, TITULO_01, TITULO_02, TITULO_03 + FROM ERGON.ERG_PM_DECISAO + """, + }, + "erg_pm_grupoexame": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + GRUPOEXAME, NOME, FLEX_CAMPO_01, FLEX_CAMPO_02, + FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05 + FROM ERGON.ERG_PM_GRUPOEXAME + """, + }, + "erg_pm_pront_med": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + PROFISPRONT, CHAVEPRONT, CRM, PROFISSIONAL, SENHA, + OK, NOK, JUSTIFICATIVA, "DATA", ORDEM_HOMOL, FLEX_CAMPO_01, + FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05, + LEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10 + FROM ERGON.ERG_PM_PRONT_MED + """, + }, + "erg_pm_resultpront": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + RESULTPRONT, CHAVEPRONT, DECISAO, NUMVINC, DTINI, + DTFIM, NUMDIAS, TIPOFREQ, CODFREQ, PONTPUBL, DTCONCL, + ERROCONCL, RESULTRETIF, FLEX_CAMPO_01, FLEX_CAMPO_02, + FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05, FLEX_CAMPO_06, + FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10, + JUSTIFICATIVA, PRAZO, NEXO, JURIDICO, MOTIVOPUBL, VERSAOPUBL, + FLEX_CAMPO_11 + FROM ERGON.ERG_PM_RESULTPRONT; + """, + }, + "exames": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + SIGLA, NOME, FLEX_CAMPO_01, FLEX_CAMPO_02, FLEX_CAMPO_03, + FLEX_CAMPO_04, FLEX_CAMPO_05, CONSULTAS_AFAST, PRORROG_LIC, + OBRIG_ESPEC, OBRIG_ULTATEND, PRAZO_RECONSIDERACAO, QUANTMED, + GRUPOEXAME, OBRIG_DEPEN, OBRIG_CAT, OBRIG_PROCESSO, OBRIG_PENS, + DISPONIVEL_UNID, TIPOEQUIPE, EXAMERECURSO, MAX_DIAS_RETROAC, + VAGASAGENDA, IMPRIME_CONVOCACAO, SETOR_CONVOCACAO, INSTRUCOES, + FLEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, + FLEX_CAMPO_10, FICHA_VACINA, EXAME_LABORATORIO, EXAME_COMPLEMENTAR, + TIPO_PERICIA, INGRESSO, PUBLICA_AGENDAMENTO, DATA_AGENDAMENTO, + FLEX_CAMPO_11, FLEX_CAMPO_12, FLEX_CAMPO_13, FLEX_CAMPO_14, FLEX_CAMPO_15, + FLEX_CAMPO_16, FLEX_CAMPO_17, FLEX_CAMPO_18, FLEX_CAMPO_19, FLEX_CAMPO_20, + PERICIA_EXTERNA, GRUPO2EXAME, TITULO_01, TITULO_02, TITULO_03, LEMBRETE + FROM ERGON.EXAMES + """, + }, + "funcoes_ev": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + FUNCAO, NOME, CONTROLE_VAGA, FLEX_CAMPO_01, FLEX_CAMPO_02, + FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05, PONTPUBL, + FLEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, + FLEX_CAMPO_10, FLEX_CAMPO_11, FLEX_CAMPO_12, FLEX_CAMPO_13, FLEX_CAMPO_14, + FLEX_CAMPO_15, FLEX_CAMPO_16, FLEX_CAMPO_17, FLEX_CAMPO_18, FLEX_CAMPO_19, + FLEX_CAMPO_20, FLEX_CAMPO_21, FLEX_CAMPO_22, FLEX_CAMPO_23, FLEX_CAMPO_24, + FLEX_CAMPO_25, FLEX_CAMPO_26, FLEX_CAMPO_27, FLEX_CAMPO_28, FLEX_CAMPO_29, + FLEX_CAMPO_30, DT_INICIO_CONTR_VAGA, ID_REG + FROM ERGON.FUNCOES_EV_ + """, + }, + "medicos": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + CRM, NOME, NUMFUNC, FLEX_CAMPO_01, FLEX_CAMPO_02, FLEX_CAMPO_03, + FLEX_CAMPO_04, FLEX_CAMPO_05, DATAINATIV, ID_PESSOA, FLEX_CAMPO_06, + FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10, FLEX_CAMPO_11, + FLEX_CAMPO_12, FLEX_CAMPO_13, FLEX_CAMPO_14, FLEX_CAMPO_15 + FROM ERGON.MEDICOS + """, + }, + "prontuario": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + NUMFUNC, SIGLA, DT_HOMOL, CRM_HOMOL, + CRM_ATEST, COD_CID1, TP_CID1, COD_CID2, + TP_CID2, COD_CID3, TP_CID3, ATEST_PUBL, + TINI, DTFIM, APTO, TEXTO_AUT, TEXTO_MEDICO, + TIPO_LIC, EXAM_TEXT, COD_ACIDENTE, FLEX_CAMPO_01, + FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, + FLEX_CAMPO_05, FLEX_CAMPO_06, FLEX_CAMPO_07, + FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10, + CHAVE, PONTPUBL, CHAVEATEND, DTATEND, HORAATEND, + FLEX_CAMPO_11, FLEX_CAMPO_12, FLEX_CAMPO_13, + FLEX_CAMPO_14, FLEX_CAMPO_15, FLEX_CAMPO_16, + FLEX_CAMPO_17, FLEX_CAMPO_18, FLEX_CAMPO_19, FLEX_CAMPO_20, + FLEX_CAMPO_21, FLEX_CAMPO_22, FLEX_CAMPO_23, FLEX_CAMPO_24, FLEX_CAMPO_25, + FLEX_CAMPO_26, FLEX_CAMPO_27, FLEX_CAMPO_28, FLEX_CAMPO_29, FLEX_CAMPO_30, + ANAMNESE, EXAMEADIC, DTCONCLUSAO, REQPERICIA, CHAVEANT, CAT, NOMEPARENTE, + GRAU_PARENTESCO, NUMPENS, NUMVINC_PENS, NUMPROC, US, ID_REG, INSCRITO, + JUSTIFICATIVA, NUMDEP, FLEX_CAMPO_31, FLEX_CAMPO_32, FLEX_CAMPO_33, + FLEX_CAMPO_34, FLEX_CAMPO_35 + FROM ERGON.PRONTUARIO + """, + }, + "tpmrj_pm_juntas": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + ID_JUNTA, NUMFUNC, SIGLA, DTINI, + JUSTIFICATIVA, DTCONCLUSAO, NUMPROC + FROM C_ERGON.TPMRJ_PM_JUNTAS + """, + }, + "tpmrj_pm_periciasjunta": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + ID_PERICIA_JUNTA, ID_JUNTA, CHAVEPRONT + FROM C_ERGON.TPMRJ_PM_PERICIASJUNTA + """, + }, + "tpmrj_pm_resultjunta": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + ID_RESULTJUNTA, ID_JUNTA, DECISAO, NUMVINC, + DTINI, DTFIM, NUMDIAS, ID_RESULTJUNTARETIF, + PONTPUBL, JUSTIF_RETIF, RETIFICADO, TIPOFREQ, + CODFREQ, CRM, PROFISSIONAL, COMPL_DECISAO_PUBL, + ERROCONCL, LOTE_PUBL + FROM C_ERGON.TPMRJ_PM_RESULTJUNTA + """, + }, + "tpmrj_pm_vwbim ": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + ROWID_REG, ID_BIM, ID_SOLIC_BIM, NUMFUNC, + NUMVINC, MATRIC, NOME, TIPOVINC, CARGO, + DESC_CARGO, SETOR, NOMESETOR, READAPTADO, + TIPO_PESSOA, NUMDEP, DT_SOLICITACAO, REQUERIMENTO, + EH_REASSUNCAO, EMAIL, TELEFONE, NOME_DEPENDENTE, + MOTIVO_INSPECAO, SITUACAO, SITUACAO_DESC, FALTANDO_SERVICO, + FALTANDO_SERVICO_DESDE, SITUACAO_FUNCIONAL_REGULAR, + HORARIO_TRABALHO_INI, HORARIO_TRABALHO_FIM, OBSERVACAO_RH, + MOTIVO_REJEICAO, ARTIGO, DIAS_DIF_AGENDAMENTO + FROM C_ERGON.TPMRJ_PM_VWBIM + """, + }, + "tpmrj_pm_vwtpconclper ": { + "materialize_after_dump": True, + "biglake_table": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + ROWID_REG, ID_SOLIC_BIM, NUMFUNC, + NUMVINC, MATRICULA, NOME, DT_SOLICITACAO, + REQUERIMENTO, REQUERIMENTO_DESC, EH_REASSUNCAO, + EMAIL, EMAIL_FUNCIONARIO, TELEFONE, + TELEFONE_FUNCIONARIO, EMAIL_CHEFE, TELEFONE_CHEFE, + TIPO_PESSOA, MOTIVO_INSPECAO, EXAME_PESSOA_DE, NUMDEP, + NOME_DEPENDENTE, GRAU_PARENTESCO, COD_SETOR, SETOR, + SITUACAO, DESC_SITUACAO, ORIGEM_SOLIC, E_MAIL_ALTERNATIVO, + EH_EMAIL_INSTITUCIONAL, EH_EMAIL_INSTITUCIONAL_BIM + FROM C_ERGON.TPMRJ_PM_VWSOLICBIM """, }, } + ergon_clocks = generate_dump_db_schedules( interval=timedelta(days=1), start_date=datetime(2022, 11, 9, 22, 30, tzinfo=pytz.timezone("America/Sao_Paulo")), @@ -53,3 +302,13 @@ ) ergon_pericia_medica_monthly_update_schedule = Schedule(clocks=untuple(ergon_clocks)) + + +# for table_id in ergon_queries: +# query = f"""SELECT +# * +# FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.{table_id}` +# """ +# # save the query in a table_id.sql +# with open(f"queries/models/recursos_humanos_ergon_pericia_medica/{table_id}.sql", "w") as f: +# f.write(query.replace(" ", "")) diff --git a/queries/models/recursos_humanos_ergon/rubrica.sql b/queries/models/recursos_humanos_ergon/rubrica.sql new file mode 100644 index 0000000..9abebe3 --- /dev/null +++ b/queries/models/recursos_humanos_ergon/rubrica.sql @@ -0,0 +1,2 @@ +SELECT * +FROM rj-smfp.recursos_humanos_ergon_staging.rubrica AS t \ No newline at end of file diff --git a/queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql b/queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql deleted file mode 100644 index 7f077bb..0000000 --- a/queries/models/recursos_humanos_ergon_folha_pagamento/folha_pagamento.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT * -FROM rj-smfp.recursos_humanos_ergon_folha_pagamento_staging.folha_pagamento AS t \ No newline at end of file diff --git a/queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql b/queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql deleted file mode 100644 index a77fb57..0000000 --- a/queries/models/recursos_humanos_ergon_folha_pagamento/pericia_medica.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT * -FROM rj-smfp.recursos_humanos_ergon_folha_pagamento_staging.pericia_medica AS t \ No newline at end of file diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/cid.sql b/queries/models/recursos_humanos_ergon_pericia_medica/cid.sql new file mode 100644 index 0000000..c20edb4 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/cid.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.cid` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/designacoes_ev.sql b/queries/models/recursos_humanos_ergon_pericia_medica/designacoes_ev.sql new file mode 100644 index 0000000..2cdb602 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/designacoes_ev.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.designacoes_ev` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_cid10_capitulo.sql b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_cid10_capitulo.sql new file mode 100644 index 0000000..e96813b --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_cid10_capitulo.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.erg_pm_cid10_capitulo` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_decisao.sql b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_decisao.sql new file mode 100644 index 0000000..14cf85f --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_decisao.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.erg_pm_decisao` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_grupoexame.sql b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_grupoexame.sql new file mode 100644 index 0000000..bec5712 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_grupoexame.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.erg_pm_grupoexame` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_pront_med.sql b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_pront_med.sql new file mode 100644 index 0000000..9762af8 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_pront_med.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.erg_pm_pront_med` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_resultpront.sql b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_resultpront.sql new file mode 100644 index 0000000..227b3e7 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/erg_pm_resultpront.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.erg_pm_resultpront` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/exames.sql b/queries/models/recursos_humanos_ergon_pericia_medica/exames.sql new file mode 100644 index 0000000..8a0f34e --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/exames.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.exames` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/funcoes_ev.sql b/queries/models/recursos_humanos_ergon_pericia_medica/funcoes_ev.sql new file mode 100644 index 0000000..1cad946 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/funcoes_ev.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.funcoes_ev` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/medicos.sql b/queries/models/recursos_humanos_ergon_pericia_medica/medicos.sql new file mode 100644 index 0000000..32a74b1 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/medicos.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.medicos` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/prontuario.sql b/queries/models/recursos_humanos_ergon_pericia_medica/prontuario.sql new file mode 100644 index 0000000..24e0f16 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/prontuario.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.prontuario` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_juntas.sql b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_juntas.sql new file mode 100644 index 0000000..8ea63b6 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_juntas.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.tpmrj_pm_juntas` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_periciasjunta.sql b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_periciasjunta.sql new file mode 100644 index 0000000..5ec39e2 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_periciasjunta.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.tpmrj_pm_periciasjunta` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_resultjunta.sql b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_resultjunta.sql new file mode 100644 index 0000000..3a6ef6c --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_resultjunta.sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.tpmrj_pm_resultjunta` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim .sql b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim .sql new file mode 100644 index 0000000..a19d842 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim .sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.tpmrj_pm_vwbim ` diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper .sql b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper .sql new file mode 100644 index 0000000..a47dc62 --- /dev/null +++ b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper .sql @@ -0,0 +1,3 @@ +SELECT + * +FROM `rj-smfp.recursos_humanos_ergon_pericia_medica_staging.tpmrj_pm_vwtpconclper ` From 7ee14d878640f955dc598ccbf3ff2d35e68d55d0 Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 10 May 2024 13:41:20 -0300 Subject: [PATCH 3/9] chore: remove recursos_humanos_ergon_folha_pagamento from dbt --- queries/dbt_project.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index d3c4517..8100ac7 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -48,9 +48,6 @@ models: compras_materiais_servicos_sigma: +materialized: table +schema: compras_materiais_servicos_sigma - recursos_humanos_ergon_folha_pagamento: - +materialized: table - +schema: recursos_humanos_ergon_folha_pagamento recursos_humanos_ergon_pericia_medica: +materialized: table +schema: recursos_humanos_ergon_pericia_medica From 1d51bdaada18e67b87e3ccdf5cf0e18561730d1b Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 10 May 2024 14:04:40 -0300 Subject: [PATCH 4/9] fix: model file name --- .../{tpmrj_pm_vwbim .sql => tpmrj_pm_vwbim.sql} | 0 .../{tpmrj_pm_vwtpconclper .sql => tpmrj_pm_vwtpconclper.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename queries/models/recursos_humanos_ergon_pericia_medica/{tpmrj_pm_vwbim .sql => tpmrj_pm_vwbim.sql} (100%) rename queries/models/recursos_humanos_ergon_pericia_medica/{tpmrj_pm_vwtpconclper .sql => tpmrj_pm_vwtpconclper.sql} (100%) diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim .sql b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim.sql similarity index 100% rename from queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim .sql rename to queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwbim.sql diff --git a/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper .sql b/queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper.sql similarity index 100% rename from queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper .sql rename to queries/models/recursos_humanos_ergon_pericia_medica/tpmrj_pm_vwtpconclper.sql From 117f1c41aa1e807b86272aa0d44d44d6f3ea0266 Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 10 May 2024 14:25:10 -0300 Subject: [PATCH 5/9] fix: db configuration --- Dockerfile | 24 +++- openssl.cnf | 401 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+), 4 deletions(-) create mode 100644 openssl.cnf diff --git a/Dockerfile b/Dockerfile index e79e4a2..1824850 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,35 @@ # Build arguments ARG PYTHON_VERSION=3.10-slim-buster +# Get Oracle Instant Client +FROM curlimages/curl:7.81.0 as curl-step +ARG ORACLE_INSTANT_CLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/215000/instantclient-basic-linux.x64-21.5.0.0.0dbru.zip +RUN curl -sSLo /tmp/instantclient.zip $ORACLE_INSTANT_CLIENT_URL + +# Unzip Oracle Instant Client +FROM ubuntu:18.04 as unzip-step +COPY --from=curl-step /tmp/instantclient.zip /tmp/instantclient.zip +RUN apt-get update && \ + apt-get install --no-install-recommends -y unzip && \ + rm -rf /var/lib/apt/lists/* && \ + unzip /tmp/instantclient.zip -d /tmp + # Start Python image FROM python:${PYTHON_VERSION} - -# Install a few dependencies +# Install a few dependencies and setup oracle instant client +WORKDIR /opt/oracle +COPY --from=unzip-step /tmp/instantclient_21_5 /opt/oracle/instantclient_21_5 RUN apt-get update && \ apt-get install --no-install-recommends -y git curl gnupg2 libaio1 && \ curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ echo "deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list && \ apt-get update && \ - ACCEPT_EULA=Y apt-get install -y git msodbcsql17 openssl unixodbc-dev && \ + ACCEPT_EULA=Y apt-get install --no-install-recommends -y ffmpeg libsm6 libxext6 msodbcsql17 openssl unixodbc-dev && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* && \ + sh -c "echo /opt/oracle/instantclient_21_5 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ + ldconfig COPY ./openssl.cnf /etc/ssl/openssl.cnf # Setting environment with prefect version diff --git a/openssl.cnf b/openssl.cnf new file mode 100644 index 0000000..9a76ed4 --- /dev/null +++ b/openssl.cnf @@ -0,0 +1,401 @@ +# +# OpenSSL example configuration file. +# See doc/man5/config.pod for more info. +# +# This is mostly being used for generation of certificate requests, +# but may be used for auto loading of providers + +# Note that you can include other files from the main configuration +# file using the .include directive. +#.include filename +openssl_conf = default_conf + +# This definition stops the following lines choking if HOME isn't +# defined. +HOME = . + + # Use this in order to automatically load providers. +openssl_conf = openssl_init + +# Comment out the next line to ignore configuration errors +config_diagnostics = 1 + +# Extra OBJECT IDENTIFIER info: +# oid_file = $ENV::HOME/.oid +oid_section = new_oids + +# To use this configuration file with the "-extfile" option of the +# "openssl x509" utility, name here the section containing the +# X.509v3 extensions to use: +# extensions = +# (Alternatively, use a configuration file that has only +# X.509v3 extensions in its main [= default] section.) + +[ new_oids ] +# We can add new OIDs in here for use by 'ca', 'req' and 'ts'. +# Add a simple OID like this: +# testoid1=1.2.3.4 +# Or use config file substitution like this: +# testoid2=${testoid1}.5.6 + +# Policies used by the TSA examples. +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +# For FIPS +# Optionally include a file that is generated by the OpenSSL fipsinstall +# application. This file contains configuration data required by the OpenSSL +# fips provider. It contains a named section e.g. [fips_sect] which is +# referenced from the [provider_sect] below. +# Refer to the OpenSSL security policy for more information. +# .include fipsmodule.cnf + +[openssl_init] +# providers = provider_sect + +# List of providers to load +# [provider_sect] +# default = default_sect +# The fips section name should match the section name inside the +# included fipsmodule.cnf. +# fips = fips_sect + +# If no providers are activated explicitly, the default one is activated implicitly. +# See man 7 OSSL_PROVIDER-default for more details. +# +# If you add a section explicitly activating any other provider(s), you most +# probably need to explicitly activate the default provider, otherwise it +# becomes unavailable in openssl. As a consequence applications depending on +# OpenSSL may not work correctly which could lead to significant system +# problems including inability to remotely access the system. +# [default_sect] +# activate = 1 + + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = ./demoCA # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several certs with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/cacert.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/private/cakey.pem# The private key + +x509_extensions = usr_cert # The extensions to add to the cert + +# Comment out the following two lines for the "traditional" +# (and highly broken) format. +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +# Extension copying option: use with caution. +# copy_extensions = copy + +# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs +# so this is commented out by default to leave a V1 CRL. +# crlnumber must also be commented out to leave a V1 CRL. +# crl_extensions = crl_ext + +default_days = 365 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = default # use public key default MD +preserve = no # keep passed DN ordering + +# A few difference way of specifying how similar the request should look +# For type CA, the listed attributes must be the same, and the optional +# and supplied fields are just that :-) +policy = policy_match + +# For the CA policy +[ policy_match ] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +# For the 'anything' policy +# At this point in time, you must list all acceptable 'object' +# types. +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca # The extensions to add to the self signed cert + +# Passwords for private keys if not present they will be prompted for +# input_password = secret +# output_password = secret + +# This sets a mask for permitted string types. There are several options. +# default: PrintableString, T61String, BMPString. +# pkix : PrintableString, BMPString (PKIX recommendation before 2004) +# utf8only: only UTF8Strings (PKIX recommendation after 2004). +# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings). +# MASK:XXXX a literal mask value. +# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings. +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = AU +countryName_min = 2 +countryName_max = 2 + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Some-State + +localityName = Locality Name (eg, city) + +0.organizationName = Organization Name (eg, company) +0.organizationName_default = Internet Widgits Pty Ltd + +# we can do this but it is not needed normally :-) +#1.organizationName = Second Organization Name (eg, company) +#1.organizationName_default = World Wide Web Pty Ltd + +organizationalUnitName = Organizational Unit Name (eg, section) +#organizationalUnitName_default = + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_max = 64 + +emailAddress = Email Address +emailAddress_max = 64 + +# SET-ex3 = SET extension number 3 + +[ req_attributes ] +challengePassword = A challenge password +challengePassword_min = 4 +challengePassword_max = 20 + +unstructuredName = An optional company name + +[ usr_cert ] + +# These extensions are added when 'ca' signs a request. + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +# This is required for TSA certificates. +# extendedKeyUsage = critical,timeStamping + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v3_ca ] + + +# Extensions for a typical CA + + +# PKIX recommendation. + +subjectKeyIdentifier=hash + +authorityKeyIdentifier=keyid:always,issuer + +basicConstraints = critical,CA:true + +# Key usage: this is typical for a CA certificate. However since it will +# prevent it being used as an test self-signed certificate it is best +# left out by default. +# keyUsage = cRLSign, keyCertSign + +# Include email address in subject alt name: another PKIX recommendation +# subjectAltName=email:copy +# Copy issuer details +# issuerAltName=issuer:copy + +# DER hex encoding of an extension: beware experts only! +# obj=DER:02:03 +# Where 'obj' is a standard or added object +# You can even override a supported extension: +# basicConstraints= critical, DER:30:03:01:01:FF + +[ crl_ext ] + +# CRL extensions. +# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. + +# issuerAltName=issuer:copy +authorityKeyIdentifier=keyid:always + +[ proxy_cert_ext ] +# These extensions should be added when creating a proxy certificate + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +# This really needs to be in place for it to be a proxy certificate. +proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo + +#################################################################### +[ tsa ] + +default_tsa = tsa_config1 # the default TSA section + +[ tsa_config1 ] + +# These are used by the TSA reply generation only. +dir = ./demoCA # TSA root directory +serial = $dir/tsaserial # The current serial number (mandatory) +crypto_device = builtin # OpenSSL engine to use for signing +signer_cert = $dir/tsacert.pem # The TSA signing certificate + # (optional) +certs = $dir/cacert.pem # Certificate chain to include in reply + # (optional) +signer_key = $dir/private/tsakey.pem # The TSA private key (optional) +signer_digest = sha256 # Signing digest to use. (Optional) +default_policy = tsa_policy1 # Policy if request did not specify it + # (optional) +other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional) +digests = sha1, sha256, sha384, sha512 # Acceptable message digests (mandatory) +accuracy = secs:1, millisecs:500, microsecs:100 # (optional) +clock_precision_digits = 0 # number of digits after dot. (optional) +ordering = yes # Is ordering defined for timestamps? + # (optional, default: no) +tsa_name = yes # Must the TSA name be included in the reply? + # (optional, default: no) +ess_cert_id_chain = no # Must the ESS cert id chain be included? + # (optional, default: no) +ess_cert_id_alg = sha1 # algorithm to compute certificate + # identifier (optional, default: sha1) + +[insta] # CMP using Insta Demo CA +# Message transfer +server = pki.certificate.fi:8700 +# proxy = # set this as far as needed, e.g., http://192.168.1.1:8080 +# tls_use = 0 +path = pkix/ + +# Server authentication +recipient = "/C=FI/O=Insta Demo/CN=Insta Demo CA" # or set srvcert or issuer +ignore_keyusage = 1 # potentially needed quirk +unprotected_errors = 1 # potentially needed quirk +extracertsout = insta.extracerts.pem + +# Client authentication +ref = 3078 # user identification +secret = pass:insta # can be used for both client and server side + +# Generic message options +cmd = ir # default operation, can be overridden on cmd line with, e.g., kur + +# Certificate enrollment +subject = "/CN=openssl-cmp-test" +newkey = insta.priv.pem +out_trusted = apps/insta.ca.crt # does not include keyUsage digitalSignature +certout = insta.cert.pem + +[pbm] # Password-based protection for Insta CA +# Server and client authentication +ref = $insta::ref # 3078 +secret = $insta::secret # pass:insta + +[signature] # Signature-based protection for Insta CA +# Server authentication +trusted = $insta::out_trusted # apps/insta.ca.crt + +# Client authentication +secret = # disable PBM +key = $insta::newkey # insta.priv.pem +cert = $insta::certout # insta.cert.pem + +[ir] +cmd = ir + +[cr] +cmd = cr + +[kur] +# Certificate update +cmd = kur +oldcert = $insta::certout # insta.cert.pem + +[rr] +# Certificate revocation +cmd = rr +oldcert = $insta::certout # insta.cert.pem + +[default_conf] +ssl_conf = ssl_sect + +[ssl_sect] +system_default = system_default_sect + +[system_default_sect] +MinProtocol = TLSv1.0 +CipherString = DEFAULT@SECLEVEL=1 \ No newline at end of file From c819d46d73da40332fcf94c2aca24a873ccce7e1 Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 10 May 2024 14:30:36 -0300 Subject: [PATCH 6/9] fix: db configuration --- pipelines/ergon/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/ergon/__init__.py b/pipelines/ergon/__init__.py index ec76c70..084eebe 100644 --- a/pipelines/ergon/__init__.py +++ b/pipelines/ergon/__init__.py @@ -1,2 +1,3 @@ # -*- coding: utf-8 -*- from pipelines.ergon.dump_db_ergon.flows import * # noqa +from pipelines.ergon.dump_db_ergon_pericia_medica.flows import * # noqa From e5123d9924ca4dd0f82c5495442e829f1feb78b7 Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 10 May 2024 14:45:55 -0300 Subject: [PATCH 7/9] fix: db configuration --- pipelines/ergon/__init__.py | 1 + pipelines/ergon/dump_db_ergon/__init__.py | 0 pipelines/ergon/dump_db_ergon_pericia_medica/__init__.py | 0 3 files changed, 1 insertion(+) create mode 100644 pipelines/ergon/dump_db_ergon/__init__.py create mode 100644 pipelines/ergon/dump_db_ergon_pericia_medica/__init__.py diff --git a/pipelines/ergon/__init__.py b/pipelines/ergon/__init__.py index 084eebe..4596fd1 100644 --- a/pipelines/ergon/__init__.py +++ b/pipelines/ergon/__init__.py @@ -1,3 +1,4 @@ # -*- coding: utf-8 -*- +## from pipelines.ergon.dump_db_ergon.flows import * # noqa from pipelines.ergon.dump_db_ergon_pericia_medica.flows import * # noqa diff --git a/pipelines/ergon/dump_db_ergon/__init__.py b/pipelines/ergon/dump_db_ergon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/__init__.py b/pipelines/ergon/dump_db_ergon_pericia_medica/__init__.py new file mode 100644 index 0000000..e69de29 From 166d4c9803f10afdfc6f6bb7e7b16008dbfefea6 Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 10 May 2024 15:06:59 -0300 Subject: [PATCH 8/9] fix: db configuration --- pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py index 3492da1..64fe3b6 100644 --- a/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py +++ b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py @@ -122,7 +122,7 @@ FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10, JUSTIFICATIVA, PRAZO, NEXO, JURIDICO, MOTIVOPUBL, VERSAOPUBL, FLEX_CAMPO_11 - FROM ERGON.ERG_PM_RESULTPRONT; + FROM ERGON.ERG_PM_RESULTPRONT """, }, "exames": { From 1a15b505b0863d244c0dd3ad2d3d95dbcf3a82d3 Mon Sep 17 00:00:00 2001 From: d116626 Date: Mon, 13 May 2024 12:07:01 -0300 Subject: [PATCH 9/9] fix: db configuration --- .../dump_db_ergon_pericia_medica/schedules.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py index 64fe3b6..68272bd 100644 --- a/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py +++ b/pipelines/ergon/dump_db_ergon_pericia_medica/schedules.py @@ -104,7 +104,7 @@ PROFISPRONT, CHAVEPRONT, CRM, PROFISSIONAL, SENHA, OK, NOK, JUSTIFICATIVA, "DATA", ORDEM_HOMOL, FLEX_CAMPO_01, FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, FLEX_CAMPO_05, - LEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10 + FLEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10 FROM ERGON.ERG_PM_PRONT_MED """, }, @@ -186,24 +186,22 @@ "dump_mode": "overwrite", "execute_query": """ SELECT - NUMFUNC, SIGLA, DT_HOMOL, CRM_HOMOL, - CRM_ATEST, COD_CID1, TP_CID1, COD_CID2, - TP_CID2, COD_CID3, TP_CID3, ATEST_PUBL, - TINI, DTFIM, APTO, TEXTO_AUT, TEXTO_MEDICO, - TIPO_LIC, EXAM_TEXT, COD_ACIDENTE, FLEX_CAMPO_01, - FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, - FLEX_CAMPO_05, FLEX_CAMPO_06, FLEX_CAMPO_07, - FLEX_CAMPO_08, FLEX_CAMPO_09, FLEX_CAMPO_10, - CHAVE, PONTPUBL, CHAVEATEND, DTATEND, HORAATEND, - FLEX_CAMPO_11, FLEX_CAMPO_12, FLEX_CAMPO_13, - FLEX_CAMPO_14, FLEX_CAMPO_15, FLEX_CAMPO_16, - FLEX_CAMPO_17, FLEX_CAMPO_18, FLEX_CAMPO_19, FLEX_CAMPO_20, - FLEX_CAMPO_21, FLEX_CAMPO_22, FLEX_CAMPO_23, FLEX_CAMPO_24, FLEX_CAMPO_25, - FLEX_CAMPO_26, FLEX_CAMPO_27, FLEX_CAMPO_28, FLEX_CAMPO_29, FLEX_CAMPO_30, - ANAMNESE, EXAMEADIC, DTCONCLUSAO, REQPERICIA, CHAVEANT, CAT, NOMEPARENTE, - GRAU_PARENTESCO, NUMPENS, NUMVINC_PENS, NUMPROC, US, ID_REG, INSCRITO, - JUSTIFICATIVA, NUMDEP, FLEX_CAMPO_31, FLEX_CAMPO_32, FLEX_CAMPO_33, - FLEX_CAMPO_34, FLEX_CAMPO_35 + NUMFUNC, SIGLA, DT_HOMOL, CRM_HOMOL, CRM_ATEST, + COD_CID1, TP_CID1, COD_CID2, TP_CID2, COD_CID3, + TP_CID3, ATEST_PUBL, DTINI, DTFIM, APTO, TEXTO_AUT, + TEXTO_MEDICO, TIPO_LIC, EXAM_TEXT, COD_ACIDENTE, + FLEX_CAMPO_01, FLEX_CAMPO_02, FLEX_CAMPO_03, FLEX_CAMPO_04, + FLEX_CAMPO_05, FLEX_CAMPO_06, FLEX_CAMPO_07, FLEX_CAMPO_08, + FLEX_CAMPO_09, FLEX_CAMPO_10, CHAVE, PONTPUBL, CHAVEATEND, + DTATEND, HORAATEND, FLEX_CAMPO_11, FLEX_CAMPO_12, FLEX_CAMPO_13, + FLEX_CAMPO_14, FLEX_CAMPO_15, FLEX_CAMPO_16, FLEX_CAMPO_17, + FLEX_CAMPO_18, FLEX_CAMPO_19, FLEX_CAMPO_20, FLEX_CAMPO_21, + FLEX_CAMPO_22, FLEX_CAMPO_23, FLEX_CAMPO_24, FLEX_CAMPO_25, + FLEX_CAMPO_26, FLEX_CAMPO_27, FLEX_CAMPO_28, FLEX_CAMPO_29, + FLEX_CAMPO_30, ANAMNESE, EXAMEADIC, DTCONCLUSAO, REQPERICIA, + CHAVEANT, CAT, NOMEPARENTE, GRAU_PARENTESCO, NUMPENS, NUMVINC_PENS, + NUMPROC, US, ID_REG, INSCRITO, JUSTIFICATIVA, NUMDEP, FLEX_CAMPO_31, + FLEX_CAMPO_32, FLEX_CAMPO_33, FLEX_CAMPO_34, FLEX_CAMPO_35 FROM ERGON.PRONTUARIO """, },