From 65983cf67fc738c8fa9e87469b3e85907d4d86e0 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 12 Sep 2024 19:08:56 -0300 Subject: [PATCH 1/7] chore: put database_get inside dump_upload_batch. made redo the connection for every query in queries --- .../dump_db_porte_empresa/schedules.py | 2 +- poetry.lock | 90 ++++++++++++++++++- pyproject.toml | 2 +- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py index 46cc73a..956f089 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Schedules for the database dump pipeline........ +Schedules for the database dump pipeline......... """ from datetime import datetime, timedelta diff --git a/poetry.lock b/poetry.lock index ad70d11..613f011 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2568,6 +2568,7 @@ jinja2 = {version = "^3.1.2", optional = true} loguru = "^0.7.0" pandas = {version = "^2.0.3", optional = true} prefect = {version = "1.4.1", optional = true} +psycopg2-binary = {version = "^2.9.9", optional = true} pymongo = {version = "^4.6.0", optional = true} pymysql = {version = "^1.1.0", optional = true} pyodbc = {version = "^5.0.1", optional = true} @@ -2582,14 +2583,14 @@ actions = ["ruamel-yaml (>=0.18.6,<0.19.0)"] all = ["aiohttp (>=3.8.5,<4.0.0)", "basedosdados[upload] (==2.0.0b21)", "croniter (>=2.0.1,<3.0.0)", "cx-oracle (>=8.3.0,<9.0.0)", "dask (>=2023.11.0,<2024.0.0)", "earthengine-api (>=0.1.379,<0.2.0)", "geojsplit (>=0.1.2,<0.2.0)", "geopandas (>=0.14.1,<0.15.0)", "geopy (>=2.4.1,<3.0.0)", "google-cloud-storage (>=2.10.0,<3.0.0)", "gspread (>=5.12.4,<6.0.0)", "infisical (>=1.4.2,<1.6)", "jinja2 (>=3.1.2,<4.0.0)", "numpy (>=1.23.3,<2.0.0)", "pandas (>=2.0.3,<3.0.0)", "pendulum (>=2.1.2,<3.0.0)", "prefect (==1.4.1)", "pymongo (>=4.6.0,<5.0.0)", "pymysql (>=1.1.0,<2.0.0)", "pyodbc (>=5.0.1,<6.0.0)", "pytz (>=2023.3,<2024.0)", "redis-pal (>=1.0.0,<2.0.0)", "requests (>=2.31.0,<3.0.0)", "ruamel-yaml (>=0.18.6,<0.19.0)", "scikit-learn (>=1.1.2,<2.0.0)", "sentry-sdk (>=1.29.2,<2.0.0)", "shapely (>=2.0.2,<3.0.0)"] metrics = ["numpy (>=1.23.3,<2.0.0)", "scikit-learn (>=1.1.2,<2.0.0)"] pipelines = ["basedosdados[upload] (==2.0.0b21)", "dask (>=2023.11.0,<2024.0.0)", "google-cloud-storage (>=2.10.0,<3.0.0)", "infisical (>=1.4.2,<1.6)", "jinja2 (>=3.1.2,<4.0.0)", "pandas (>=2.0.3,<3.0.0)", "prefect (==1.4.1)", "sentry-sdk (>=1.29.2,<2.0.0)"] -pipelines-templates = ["croniter (>=2.0.1,<3.0.0)", "cx-oracle (>=8.3.0,<9.0.0)", "earthengine-api (>=0.1.379,<0.2.0)", "geojsplit (>=0.1.2,<0.2.0)", "geopandas (>=0.14.1,<0.15.0)", "geopy (>=2.4.1,<3.0.0)", "gspread (>=5.12.4,<6.0.0)", "jinja2 (>=3.1.2,<4.0.0)", "pymongo (>=4.6.0,<5.0.0)", "pymysql (>=1.1.0,<2.0.0)", "pyodbc (>=5.0.1,<6.0.0)", "redis-pal (>=1.0.0,<2.0.0)", "requests (>=2.31.0,<3.0.0)", "shapely (>=2.0.2,<3.0.0)"] +pipelines-templates = ["croniter (>=2.0.1,<3.0.0)", "cx-oracle (>=8.3.0,<9.0.0)", "earthengine-api (>=0.1.379,<0.2.0)", "geojsplit (>=0.1.2,<0.2.0)", "geopandas (>=0.14.1,<0.15.0)", "geopy (>=2.4.1,<3.0.0)", "gspread (>=5.12.4,<6.0.0)", "jinja2 (>=3.1.2,<4.0.0)", "psycopg2-binary (>=2.9.9,<3.0.0)", "pymongo (>=4.6.0,<5.0.0)", "pymysql (>=1.1.0,<2.0.0)", "pyodbc (>=5.0.1,<6.0.0)", "redis-pal (>=1.0.0,<2.0.0)", "requests (>=2.31.0,<3.0.0)", "shapely (>=2.0.2,<3.0.0)"] sgrc = ["aiohttp (>=3.8.5,<4.0.0)", "pendulum (>=2.1.2,<3.0.0)", "pytz (>=2023.3,<2024.0)", "requests (>=2.31.0,<3.0.0)"] [package.source] type = "git" url = "https://github.com/prefeitura-rio/prefeitura-rio" -reference = "fd98e01bded04123cf162b49b0654afe6aa5bd57" -resolved_reference = "fd98e01bded04123cf162b49b0654afe6aa5bd57" +reference = "f9466e23897af3a1db9bb9fbbb1d9d5aa7193bba" +resolved_reference = "f9466e23897af3a1db9bb9fbbb1d9d5aa7193bba" [[package]] name = "proto-plus" @@ -2656,6 +2657,87 @@ files = [ [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +[[package]] +name = "psycopg2-binary" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, +] + [[package]] name = "pyarrow" version = "15.0.0" @@ -4211,4 +4293,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "c5b41661ba486b82aadf25f58d211ceb6de33f48bb4f26264d85da71553f4094" +content-hash = "cc53ec5fc60258d5ba8d6666bfe45adb42815874ab0bee79a468f87107c89368" diff --git a/pyproject.toml b/pyproject.toml index ca3a617..8655ffd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dbt-bigquery = "^1.6.1" google-cloud-storage = "^2.10.0" pendulum = "^3.0.0" prefect = "1.4.1" -prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "fd98e01bded04123cf162b49b0654afe6aa5bd57", extras = [ +prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "f9466e23897af3a1db9bb9fbbb1d9d5aa7193bba", extras = [ "pipelines", "pipelines-templates", "actions", From 8b79b93f408600083a4b5fd762fa43b74ccf94bd Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 12 Sep 2024 19:21:38 -0300 Subject: [PATCH 2/7] chore: put database_get inside dump_upload_batch. made redo the connection for every query in queries --- .../receita_federal_cnpj/dump_db_porte_empresa/flows.py | 2 +- .../receita_federal_cnpj/dump_db_porte_empresa/schedules.py | 2 +- poetry.lock | 6 +++--- pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py index 146cc26..4816ed8 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows..... +Database dumping flows...... """ from copy import deepcopy diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py index 956f089..7d5906d 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Schedules for the database dump pipeline......... +Schedules for the database dump pipeline.......... """ from datetime import datetime, timedelta diff --git a/poetry.lock b/poetry.lock index 613f011..77947c2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2589,8 +2589,8 @@ sgrc = ["aiohttp (>=3.8.5,<4.0.0)", "pendulum (>=2.1.2,<3.0.0)", "pytz (>=2023.3 [package.source] type = "git" url = "https://github.com/prefeitura-rio/prefeitura-rio" -reference = "f9466e23897af3a1db9bb9fbbb1d9d5aa7193bba" -resolved_reference = "f9466e23897af3a1db9bb9fbbb1d9d5aa7193bba" +reference = "2f3f0442ec5f60eb437830dfa50f5f897d3e5130" +resolved_reference = "2f3f0442ec5f60eb437830dfa50f5f897d3e5130" [[package]] name = "proto-plus" @@ -4293,4 +4293,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "cc53ec5fc60258d5ba8d6666bfe45adb42815874ab0bee79a468f87107c89368" +content-hash = "cec524ca6ce0bf5fba23cf9ad007007440039c103c3e5fe495757c6ee2af9d02" diff --git a/pyproject.toml b/pyproject.toml index 8655ffd..a4372bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dbt-bigquery = "^1.6.1" google-cloud-storage = "^2.10.0" pendulum = "^3.0.0" prefect = "1.4.1" -prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "f9466e23897af3a1db9bb9fbbb1d9d5aa7193bba", extras = [ +prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "2f3f0442ec5f60eb437830dfa50f5f897d3e5130", extras = [ "pipelines", "pipelines-templates", "actions", From cb7fc429ded83c34a6356bd4ed8d7980c7bb12dc Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 12 Sep 2024 19:22:49 -0300 Subject: [PATCH 3/7] chore: put database_get inside dump_upload_batch. made redo the connection for every query in queries --- pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py | 2 +- .../receita_federal_cnpj/dump_db_porte_empresa/schedules.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py index 4816ed8..d553caf 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows...... +Database dumping flows....... """ from copy import deepcopy diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py index 7d5906d..4c3035d 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Schedules for the database dump pipeline.......... +Schedules for the database dump pipeline........... """ from datetime import datetime, timedelta From f118b964c083b0cd5cfe693e3924e16969ac26ba Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 12 Sep 2024 19:23:18 -0300 Subject: [PATCH 4/7] chore: put database_get inside dump_upload_batch. made redo the connection for every query in queries --- pipelines/ergon/dump_db_ergon/schedules.py | 2 +- pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/ergon/dump_db_ergon/schedules.py b/pipelines/ergon/dump_db_ergon/schedules.py index e0554ad..06ea60a 100644 --- a/pipelines/ergon/dump_db_ergon/schedules.py +++ b/pipelines/ergon/dump_db_ergon/schedules.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa: E501 """ -Schedules for the database dump pipeline... +Schedules for the database dump pipeline.... """ from datetime import datetime, timedelta diff --git a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py index 1b582c5..14d7537 100644 --- a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py +++ b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # pylint: disable=C0302 """ -Schedules for the database dump pipeline... +Schedules for the database dump pipeline.... """ from datetime import datetime, timedelta From d5fdd0f3bb5484cb4758d85c677e0ffef3533430 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 12 Sep 2024 19:24:42 -0300 Subject: [PATCH 5/7] chore: put database_get inside dump_upload_batch. made redo the connection for every query in queries --- pipelines/ergon/dump_db_ergon/flows.py | 2 +- pipelines/ergon/dump_db_ergon/schedules.py | 2 +- pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py | 2 +- pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py | 2 +- pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py | 2 +- .../receita_federal_cnpj/dump_db_porte_empresa/schedules.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index c6b4a42..c9b8799 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project... +Database dumping flows for segovi project.... """ from copy import deepcopy diff --git a/pipelines/ergon/dump_db_ergon/schedules.py b/pipelines/ergon/dump_db_ergon/schedules.py index 06ea60a..8d428d4 100644 --- a/pipelines/ergon/dump_db_ergon/schedules.py +++ b/pipelines/ergon/dump_db_ergon/schedules.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa: E501 """ -Schedules for the database dump pipeline.... +Schedules for the database dump pipeline..... """ from datetime import datetime, timedelta diff --git a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py index 6a14732..58b4dd7 100644 --- a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py +++ b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for smfp ergon comlurb +Database dumping flows for smfp ergon comlurb. """ from copy import deepcopy diff --git a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py index 14d7537..1a079a1 100644 --- a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py +++ b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/schedules.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # pylint: disable=C0302 """ -Schedules for the database dump pipeline.... +Schedules for the database dump pipeline..... """ from datetime import datetime, timedelta diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py index d553caf..7334985 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows....... +Database dumping flows........ """ from copy import deepcopy diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py index 4c3035d..931fa98 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/schedules.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Schedules for the database dump pipeline........... +Schedules for the database dump pipeline............ """ from datetime import datetime, timedelta From 94924d3e1daee86654ad99e1a28248d64bededaf Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 12 Sep 2024 19:53:21 -0300 Subject: [PATCH 6/7] chore: start batch idx in 0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4372bf..50986c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dbt-bigquery = "^1.6.1" google-cloud-storage = "^2.10.0" pendulum = "^3.0.0" prefect = "1.4.1" -prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "2f3f0442ec5f60eb437830dfa50f5f897d3e5130", extras = [ +prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "dd8d28fcf67266f6ec7fce650c56b323e84b21a3", extras = [ "pipelines", "pipelines-templates", "actions", From 893df1b69d3013767f80282e44f65de1d3b83fc6 Mon Sep 17 00:00:00 2001 From: d116626 Date: Thu, 12 Sep 2024 19:53:58 -0300 Subject: [PATCH 7/7] chore: start batch idx in 0 --- pipelines/ergon/dump_db_ergon/flows.py | 2 +- pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py | 2 +- .../receita_federal_cnpj/dump_db_porte_empresa/flows.py | 2 +- poetry.lock | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index c9b8799..35d96e5 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project.... +Database dumping flows for segovi project..... """ from copy import deepcopy diff --git a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py index 58b4dd7..bb3ea53 100644 --- a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py +++ b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for smfp ergon comlurb. +Database dumping flows for smfp ergon comlurb.. """ from copy import deepcopy diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py index 7334985..586a8e3 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows........ +Database dumping flows......... """ from copy import deepcopy diff --git a/poetry.lock b/poetry.lock index 77947c2..1887334 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2589,8 +2589,8 @@ sgrc = ["aiohttp (>=3.8.5,<4.0.0)", "pendulum (>=2.1.2,<3.0.0)", "pytz (>=2023.3 [package.source] type = "git" url = "https://github.com/prefeitura-rio/prefeitura-rio" -reference = "2f3f0442ec5f60eb437830dfa50f5f897d3e5130" -resolved_reference = "2f3f0442ec5f60eb437830dfa50f5f897d3e5130" +reference = "dd8d28fcf67266f6ec7fce650c56b323e84b21a3" +resolved_reference = "dd8d28fcf67266f6ec7fce650c56b323e84b21a3" [[package]] name = "proto-plus" @@ -4293,4 +4293,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "cec524ca6ce0bf5fba23cf9ad007007440039c103c3e5fe495757c6ee2af9d02" +content-hash = "1a63c3da0bb1b7963c01b2a050dc447ec99d6663010c6ecbe316216940eace22"