Merge pull request #716 from basedosdados/staging/br_rf_cno

[dbt] br_rf_cno

dbt_project.yml

@@ -318,6 +318,9 @@ models:
  br_rf_cafir:
  +materialized: table
  +schema: br_rf_cafir
+ br_rf_cno:
+ +materialized: table
+ +schema: br_rf_cno
  br_rj_isp_estatisticas_seguranca:
  +materialized: table
  +schema: br_rj_isp_estatisticas_seguranca

macros/custom_get_where_subquery.sql

@@ -44,6 +44,25 @@
  {% endif %}
  {% endif %}
 
+ {% if "__most_recent_date_cno__" in where %}
+ {% set max_date_query = (
+ "select max(data_extracao) as max_date from " ~ relation
+ ) %}
+ {% set max_date_result = run_query(max_date_query) %}
+ {% if execute and max_date_result.rows[0][0] %}
+ {% set max_date = max_date_result.rows[0][0] %}
+ {% set where = where | replace(
+ "__most_recent_date_cno__",
+ "data_extracao = '" ~ max_date ~ "'",
+ ) %}
+ {% do log(
+ "The test will filter by the most recent date: "
+ ~ max_date,
+ info=True,
+ ) %}
+ {% endif %}
+ {% endif %}
+
  {# This block looks for __most_recent_year_month__ placeholder #}
  {% if "__most_recent_year_month__" in where %}
  {% set max_date_query = (

models/br_rf_cno/br_rf_cno__areas.sql

@@ -0,0 +1,23 @@
+{{
+ config(
+ alias="areas",
+ schema="br_rf_cno",
+ materialized="incremental",
+ partition_by={
+ "field": "data_extracao",
+ "data_type": "date",
+ },
+ )
+}}
+
+select
+ safe_cast(data as date) data_extracao,
+ safe_cast(id_cno as string) id_cno,
+ safe_cast(categoria as string) categoria,
+ safe_cast(destinacao as string) destinacao,
+ safe_cast(tipo_obra as string) tipo_obra,
+ safe_cast(tipo_area as string) tipo_area,
+ safe_cast(tipo_area_complementar as string) tipo_area_complementar,
+ safe_cast(metragem as float64) metragem,
+from `basedosdados-staging.br_rf_cno_staging.areas` as t
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}

models/br_rf_cno/br_rf_cno__cnaes.sql

@@ -0,0 +1,18 @@
+{{
+ config(
+ alias="cnaes",
+ schema="br_rf_cno",
+ materialized="incremental",
+ partition_by={
+ "field": "data_extracao",
+ "data_type": "date",
+ },
+ )
+}}
+select
+ safe_cast(data as date) data_extracao,
+ safe_cast(data_registro as date) data_registro,
+ safe_cast(id_cno as string) id_cno,
+ safe_cast(cnae_2_subclasse as string) cnae_2_subclasse,
+from `basedosdados-staging.br_rf_cno_staging.cnaes` as t
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}

models/br_rf_cno/br_rf_cno__dicionario.sql

@@ -0,0 +1,8 @@
+{{ config(alias="dicionario", schema="br_rf_cno", materialized="table") }}
+select
+ safe_cast(id_tabela as string) id_tabela,
+ safe_cast(nome_coluna as string) nome_coluna,
+ safe_cast(chave as string) chave,
+ safe_cast(cobertura_temporal as string) cobertura_temporal,
+ safe_cast(valor as string) valor,
+from `basedosdados-staging.br_rf_cno_staging.dicionario` as t

models/br_rf_cno/br_rf_cno__microdados.sql

@@ -0,0 +1,48 @@
+{{
+ config(
+ alias="microdados",
+ schema="br_rf_cno",
+ materialized="incremental",
+ partition_by={
+ "field": "data_extracao",
+ "data_type": "date",
+ },
+ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+ )
+}}
+
+select
+ safe_cast(data as date) data_extracao,
+ safe_cast(data_situacao as date) data_situacao,
+ safe_cast(data_registro as date) data_registro,
+ safe_cast(data_inicio as date) data_inicio,
+ safe_cast(data_inicio_responsabilidade as date) data_inicio_responsabilidade,
+ safe_cast(id_pais as string) id_pais,
+ safe_cast(nome_pais as string) nome_pais,
+ safe_cast(sigla_uf as string) sigla_uf,
+ safe_cast(b.id_municipio as string) id_municipio,
+ safe_cast(id_cno as string) id_cno,
+ safe_cast(id_cno_vinculado as string) id_cno_vinculado,
+ safe_cast(ltrim(situacao, '0') as string) situacao,
+ safe_cast(id_responsavel as string) id_responsavel,
+ safe_cast(nome_responsavel as string) nome_responsavel,
+ safe_cast(ltrim(qualificacao_responsavel, '0') as string) qualificacao_responsavel,
+ safe_cast(nome_empresarial as string) nome_empresarial,
+ safe_cast(area as float64) area,
+ safe_cast(unidade_medida as string) unidade_medida,
+ safe_cast(bairro as string) bairro,
+ safe_cast(cep as string) cep,
+ safe_cast(logradouro as string) logradouro,
+ safe_cast(tipo_logradouro as string) tipo_logradouro,
+ safe_cast(numero_logradouro as string) numero_logradouro,
+ safe_cast(complemento as string) complemento,
+ safe_cast(caixa_postal as string) caixa_postal,
+from `basedosdados-staging.br_rf_cno_staging.microdados` microdados
+left join
+ (
+ select id_municipio, id_municipio_rf
+ from `basedosdados.br_bd_diretorios_brasil.municipio`
+ ) b
+ on ltrim(microdados.id_municipio_rf, '0') = b.id_municipio_rf
+
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}

models/br_rf_cno/br_rf_cno__vinculos.sql

@@ -0,0 +1,27 @@
+{{
+ config(
+ alias="vinculos",
+ schema="br_rf_cno",
+ materialized="incremental",
+ partition_by={
+ "field": "data_extracao",
+ "data_type": "date",
+ },
+ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+ )
+}}
+
+
+select
+ safe_cast(data as date) data_extracao,
+ safe_cast(data_registro as date) data_registro,
+ safe_cast(data_inicio as date) data_inicio,
+ safe_cast(data_fim as date) data_fim,
+ safe_cast(id_cno as string) id_cno,
+ safe_cast(id_responsavel as string) id_responsavel,
+ safe_cast(
+ ltrim(qualificacao_contribuinte, '0') as string
+ ) qualificacao_contribuinte,
+from `basedosdados-staging.br_rf_cno_staging.vinculos` as t
+
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}