From cc48dca79b82650d23b6128a4fee1566bd7bf1bd Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Fri, 22 Sep 2023 17:40:32 +0000 Subject: [PATCH] hotfix: fix partitions, replace pm round --- .trunk/trunk.yaml | 2 +- pdm.lock | 127 +++++++++--------- pyproject.toml | 2 +- .../models/fldoe/staging/stg_fldoe__fast.sql | 34 +++-- .../tableau/rpt_tableau__miami_fast.sql | 11 +- 5 files changed, 97 insertions(+), 79 deletions(-) diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 5db521f23..4c4c4239f 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -45,7 +45,7 @@ lint: - git-diff-check - gitleaks@8.18.0 - hadolint@2.12.0 - - markdownlint@0.36.0 + - markdownlint@0.37.0 - prettier@3.0.3 - shellcheck@0.9.0 - shfmt@3.6.0 diff --git a/pdm.lock b/pdm.lock index 78df6b539..9395ba1f4 100644 --- a/pdm.lock +++ b/pdm.lock @@ -6,7 +6,7 @@ groups = ["default", "dev"] cross_platform = true static_urls = false lock_version = "4.3" -content_hash = "sha256:913d3ee163d945d6af0ba55236331ca95649f897a2564585471b26fd08a48d72" +content_hash = "sha256:08387ac2c336d542d5ee3755bf3d8a0519a59da304ae61fcb4abadb027fc3ce3" [[package]] name = "agate" @@ -302,7 +302,7 @@ files = [ [[package]] name = "dagster" -version = "1.4.14" +version = "1.4.15" summary = "Dagster is an orchestration platform for the development, production, and observation of data assets." dependencies = [ "Jinja2", @@ -335,43 +335,43 @@ dependencies = [ "watchdog>=0.8.3", ] files = [ - {file = "dagster-1.4.14-py3-none-any.whl", hash = "sha256:b36e6c7afb132e0d310b1cf2436e5542b53ac0386f9b89ac855b55f565874e9a"}, - {file = "dagster-1.4.14.tar.gz", hash = "sha256:80dec05e3f287cf35ab7f4f3a4ffe334df48ab930607de8a9b98b3af93f2cac8"}, + {file = "dagster-1.4.15-py3-none-any.whl", hash = "sha256:d733f0c27f1fc9c3b19c106533cecca10d784ba0071e595a003e2dcd3f604344"}, + {file = "dagster-1.4.15.tar.gz", hash = "sha256:8cf440b70d5ddcac53de10b14c7225b813d10153043d7382b03be71d4153c81b"}, ] [[package]] name = "dagster-airbyte" -version = "0.20.14" +version = "0.20.15" summary = "Package for integrating Airbyte with Dagster." dependencies = [ - "dagster==1.4.14", + "dagster==1.4.15", "requests", ] files = [ - {file = "dagster-airbyte-0.20.14.tar.gz", hash = "sha256:8a53f135241eb8a6f62571e3a68cf72a0bfa3e40da2e9d329a7bce97ad57e341"}, - {file = "dagster_airbyte-0.20.14-py3-none-any.whl", hash = "sha256:bb70c3c4765ef6b5813000ea766d357b7e914607164d412d9fa84cc05f106ec9"}, + {file = "dagster-airbyte-0.20.15.tar.gz", hash = "sha256:5725d0e49ed57d8739dea97383894d3ddb46eff03c82d992c5c5dc02a9726224"}, + {file = "dagster_airbyte-0.20.15-py3-none-any.whl", hash = "sha256:a627e1cb00e099c61bd3a8a4d94542cb50b531f4ebbde5c2c1366be0aa5d0bf0"}, ] [[package]] name = "dagster-cloud" -version = "1.4.14" +version = "1.4.15" summary = "" dependencies = [ - "dagster-cloud-cli==1.4.14", - "dagster==1.4.14", + "dagster-cloud-cli==1.4.15", + "dagster==1.4.15", "pex>=2.1.132", "questionary", "requests", "typer[all]", ] files = [ - {file = "dagster_cloud-1.4.14-py3-none-any.whl", hash = "sha256:8de925507bacd7eacce4af704e82b6b8b30db784cf2f18efd7d2292b3bdff581"}, - {file = "dagster_cloud-1.4.14.tar.gz", hash = "sha256:4dd2ada5a3550841c908bda4574c311c372fa2aac886b3ef6cd44a5055fece38"}, + {file = "dagster_cloud-1.4.15-py3-none-any.whl", hash = "sha256:782c8b8af2c5a3f748828ade111f2b7527424e4c85cf3bfb61fcc5c52adc45e2"}, + {file = "dagster_cloud-1.4.15.tar.gz", hash = "sha256:73ffd76aeb18df2926dc7caef17da778c57adb32470481a500cf024e0cc6c6ba"}, ] [[package]] name = "dagster-cloud-cli" -version = "1.4.14" +version = "1.4.15" summary = "" dependencies = [ "PyYAML>=5.1", @@ -382,17 +382,17 @@ dependencies = [ "typer[all]>=0.4.1", ] files = [ - {file = "dagster_cloud_cli-1.4.14-py3-none-any.whl", hash = "sha256:eca89c9ca5da7ece39974fb35eb7a55e90a5a112de3ebae391543fa9d41535f4"}, - {file = "dagster_cloud_cli-1.4.14.tar.gz", hash = "sha256:dfaa3970eb25b45affa5937d2f457497d15612d0f15979467b278d702ce8cf75"}, + {file = "dagster_cloud_cli-1.4.15-py3-none-any.whl", hash = "sha256:f3a46b93040412f0e96e2c401e5f57356877810e1d10c4bffcaf70196aaffc31"}, + {file = "dagster_cloud_cli-1.4.15.tar.gz", hash = "sha256:fa53cfc1bc170601a9d8ffcdc67e6dd97c0fb2a1e82d21afd3576c2674ca6661"}, ] [[package]] name = "dagster-dbt" -version = "0.20.14" +version = "0.20.15" summary = "A Dagster integration for dbt" dependencies = [ "Jinja2", - "dagster==1.4.14", + "dagster==1.4.15", "dbt-core<1.7", "networkx", "orjson", @@ -401,29 +401,29 @@ dependencies = [ "typer>=0.9.0", ] files = [ - {file = "dagster-dbt-0.20.14.tar.gz", hash = "sha256:5d5a20e56d5bf1822bab09922dcbf21ef41b7317a25b484d0c7fd4ca685e32bd"}, - {file = "dagster_dbt-0.20.14-py3-none-any.whl", hash = "sha256:5513499ec5f97186b351a3a0593d9f615bfda9ffe60e8f13194395c350a6accf"}, + {file = "dagster-dbt-0.20.15.tar.gz", hash = "sha256:ed8203c3be202aa294a7e145e3a2c6dffcfd330674c625daa738e510986f3fa7"}, + {file = "dagster_dbt-0.20.15-py3-none-any.whl", hash = "sha256:3441ded1c74a8842345b376d4f0581669f0f9cd95c5749c06c669bdb0d176082"}, ] [[package]] name = "dagster-fivetran" -version = "0.20.14" +version = "0.20.15" summary = "Package for integrating Fivetran with Dagster." dependencies = [ - "dagster==1.4.14", + "dagster==1.4.15", ] files = [ - {file = "dagster-fivetran-0.20.14.tar.gz", hash = "sha256:11154e4ef505622490d76ee95a5bde44bec11d7542e57095a8b5d4bfb68be7eb"}, - {file = "dagster_fivetran-0.20.14-py3-none-any.whl", hash = "sha256:d73dd10a8b0b756a68fe6274d25567aafbad9d989a1f10f956cbe018162721ed"}, + {file = "dagster-fivetran-0.20.15.tar.gz", hash = "sha256:091796f586134dcdc3c5763cba1ff096b5b6abe03f7fdf885cf3f4e69a1d5f6a"}, + {file = "dagster_fivetran-0.20.15-py3-none-any.whl", hash = "sha256:da55fa6e346e5af97e9636c966f0b408bfedb3fa6f5ae2012e5dd07015ce128a"}, ] [[package]] name = "dagster-gcp" -version = "0.20.14" +version = "0.20.15" summary = "Package for GCP-specific Dagster framework op and resource components." dependencies = [ - "dagster-pandas==0.20.14", - "dagster==1.4.14", + "dagster-pandas==0.20.15", + "dagster==1.4.15", "db-dtypes", "google-api-python-client", "google-cloud-bigquery", @@ -431,65 +431,64 @@ dependencies = [ "oauth2client", ] files = [ - {file = "dagster-gcp-0.20.14.tar.gz", hash = "sha256:3aed3cbc675eb2685878f87ccc38b6bde792419748c54714adf1f2e7f850b024"}, - {file = "dagster_gcp-0.20.14-py3-none-any.whl", hash = "sha256:47492534bd8289d571c07298bf0d3be98decd77c274bff641e4912066d7eea0b"}, + {file = "dagster-gcp-0.20.15.tar.gz", hash = "sha256:4e5cb61719924d0d6d5a15fe19f6fa2d64944258e1223d6e33e472e722ab5fbb"}, + {file = "dagster_gcp-0.20.15-py3-none-any.whl", hash = "sha256:d89747881a5b9439207359282b3af4afbe3d084207212367edcfe72e077424ec"}, ] [[package]] name = "dagster-graphql" -version = "1.4.14" +version = "1.4.15" summary = "The GraphQL frontend to python dagster." dependencies = [ - "dagster==1.4.14", + "dagster==1.4.15", "gql[requests]>=3.0.0", "graphene>=3", "requests", "starlette", - "urllib3<2.0.0", ] files = [ - {file = "dagster-graphql-1.4.14.tar.gz", hash = "sha256:db8213d8750e863b27124f618ac7b7acdf64490b55d663a4b20a149f9915b076"}, - {file = "dagster_graphql-1.4.14-py3-none-any.whl", hash = "sha256:1eb59bc4b36258a0802bab0aeeb0d87e52f6ef37fd66ee9bf42152439ca250ff"}, + {file = "dagster-graphql-1.4.15.tar.gz", hash = "sha256:c3d65cf79d2c5cf708fc471de46bd15a2663e51fec6da0eafc555d261ee8e5d1"}, + {file = "dagster_graphql-1.4.15-py3-none-any.whl", hash = "sha256:51d75862381664328f5b83dd056e503011b4a37326d8f7a635e51d7155aedd3c"}, ] [[package]] name = "dagster-k8s" -version = "0.20.14" +version = "0.20.15" summary = "A Dagster integration for k8s" dependencies = [ - "dagster==1.4.14", + "dagster==1.4.15", "kubernetes", ] files = [ - {file = "dagster-k8s-0.20.14.tar.gz", hash = "sha256:947626673063f217970f05b7ef26acf071d00f2ab357a9e442987b4d0906c0b3"}, - {file = "dagster_k8s-0.20.14-py3-none-any.whl", hash = "sha256:c5fabcbccc918b10426a1e9152599ff1836147f1f33a8a162e0248cdb1876a9a"}, + {file = "dagster-k8s-0.20.15.tar.gz", hash = "sha256:2e692d65c47b8292b2a276ec8ff6aac24f4c7bf57028cf2613db51c90613f55a"}, + {file = "dagster_k8s-0.20.15-py3-none-any.whl", hash = "sha256:412f93076cdf9e097064abcb9447ab53a5ab5eb217e93a700bf10b3b7b8b7c75"}, ] [[package]] name = "dagster-pandas" -version = "0.20.14" +version = "0.20.15" summary = "Utilities and examples for working with pandas and dagster, an opinionated framework for expressing data pipelines" dependencies = [ - "dagster==1.4.14", + "dagster==1.4.15", "pandas", ] files = [ - {file = "dagster-pandas-0.20.14.tar.gz", hash = "sha256:1aa4feddf4ed24565b7922d570e587873bde43f442f9d05274ad02a2cc3b0da0"}, - {file = "dagster_pandas-0.20.14-py3-none-any.whl", hash = "sha256:96805d06f7cd6fca1b9eb401c523b8b202db9ebd3f2fa5de6e7b2d6cb4a473f7"}, + {file = "dagster-pandas-0.20.15.tar.gz", hash = "sha256:a0607fe778dcf9a7ac0ce5493205ab8cb87092c2c0ec9413c5bdce0841150c1f"}, + {file = "dagster_pandas-0.20.15-py3-none-any.whl", hash = "sha256:6ac1d4de4252621343fbaad57a4919a0b6415f6e7b6e3cdd0b8adb79ff7dabce"}, ] [[package]] name = "dagster-ssh" -version = "0.20.14" +version = "0.20.15" summary = "Package for ssh Dagster framework components." dependencies = [ - "dagster==1.4.14", + "dagster==1.4.15", "paramiko", "sshtunnel", ] files = [ - {file = "dagster-ssh-0.20.14.tar.gz", hash = "sha256:99c3cceb154dc2883dc20506775819a093e1d2236b1eb4125bce2d9bda988dee"}, - {file = "dagster_ssh-0.20.14-py3-none-any.whl", hash = "sha256:edc8af2c0212be74b7f846550e064a27b17d111e42b8c626264bba6eb9e56608"}, + {file = "dagster-ssh-0.20.15.tar.gz", hash = "sha256:48cdb73327274668aa009a6c028333f73d5ec821f99ed8cc6e5fc566cf6bf6c2"}, + {file = "dagster_ssh-0.20.15-py3-none-any.whl", hash = "sha256:8a31e8f9f65083fe2d55010fc698fda5a7c2e2a8b038741a5dbbcf0e89420312"}, ] [[package]] @@ -526,7 +525,7 @@ files = [ [[package]] name = "dbt-core" -version = "1.6.2" +version = "1.6.3" requires_python = ">=3.8" summary = "With dbt, data analysts and engineers can build analytics the way engineers build applications." dependencies = [ @@ -555,8 +554,8 @@ dependencies = [ "urllib3~=1.0", ] files = [ - {file = "dbt-core-1.6.2.tar.gz", hash = "sha256:26915958e667b46e966d4658a74f6d20b84bd905f1db787bdfc2861267a27215"}, - {file = "dbt_core-1.6.2-py3-none-any.whl", hash = "sha256:92435746bd317b955788290baaf1c0e2729c4cc87be5878780c1f2b56219198e"}, + {file = "dbt-core-1.6.3.tar.gz", hash = "sha256:2a165425457dd679bf22bb823d45d2ba5148cda96eac8220a358718c7546073c"}, + {file = "dbt_core-1.6.3-py3-none-any.whl", hash = "sha256:4dc84d6d5262204bccb9b5f552914f9fb17ab19aca8d3a4ca5862faca66f53f7"}, ] [[package]] @@ -794,7 +793,7 @@ files = [ [[package]] name = "google-cloud-dataproc" -version = "5.5.1" +version = "5.6.0" requires_python = ">=3.7" summary = "Google Cloud Dataproc API client library" dependencies = [ @@ -804,25 +803,25 @@ dependencies = [ "protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.19.5", ] files = [ - {file = "google-cloud-dataproc-5.5.1.tar.gz", hash = "sha256:fad7da59d391fec15f1a287bd71d55ce7237819daab90e21d6b1140851d9d434"}, - {file = "google_cloud_dataproc-5.5.1-py2.py3-none-any.whl", hash = "sha256:d4b5cf3fbb40d78707c0ad49f096b8a6dd005980ca87871d3582a2ae0c0d8f6f"}, + {file = "google-cloud-dataproc-5.6.0.tar.gz", hash = "sha256:5fa9dfb88433c2d0c138b7ebb6ade174103be3762ccd039b24d53c7448e3760d"}, + {file = "google_cloud_dataproc-5.6.0-py2.py3-none-any.whl", hash = "sha256:d76a7791d5b81b498cf353be4f14a2194a696fdd1a41a69b33ac3319c5a8a213"}, ] [[package]] name = "google-cloud-storage" -version = "2.10.0" +version = "2.11.0" requires_python = ">=3.7" summary = "Google Cloud Storage API client library" dependencies = [ "google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5", "google-auth<3.0dev,>=1.25.0", "google-cloud-core<3.0dev,>=2.3.0", - "google-resumable-media>=2.3.2", + "google-resumable-media>=2.6.0", "requests<3.0.0dev,>=2.18.0", ] files = [ - {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, - {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, + {file = "google-cloud-storage-2.11.0.tar.gz", hash = "sha256:6fbf62659b83c8f3a0a743af0d661d2046c97c3a5bfb587c4662c4bc68de3e31"}, + {file = "google_cloud_storage-2.11.0-py2.py3-none-any.whl", hash = "sha256:88cbd7fb3d701c780c4272bc26952db99f25eb283fb4c2208423249f00b5fe53"}, ] [[package]] @@ -1477,7 +1476,7 @@ files = [ [[package]] name = "pandas" -version = "2.1.0" +version = "2.1.1" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" dependencies = [ @@ -1487,13 +1486,13 @@ dependencies = [ "tzdata>=2022.1", ] files = [ - {file = "pandas-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40dd20439ff94f1b2ed55b393ecee9cb6f3b08104c2c40b0cb7186a2f0046242"}, - {file = "pandas-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d4f38e4fedeba580285eaac7ede4f686c6701a9e618d8a857b138a126d067f2f"}, - {file = "pandas-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e6a0fe052cf27ceb29be9429428b4918f3740e37ff185658f40d8702f0b3e09"}, - {file = "pandas-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d81e1813191070440d4c7a413cb673052b3b4a984ffd86b8dd468c45742d3cc"}, - {file = "pandas-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eb20252720b1cc1b7d0b2879ffc7e0542dd568f24d7c4b2347cb035206936421"}, - {file = "pandas-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:38f74ef7ebc0ffb43b3d633e23d74882bce7e27bfa09607f3c5d3e03ffd9a4a5"}, - {file = "pandas-2.1.0.tar.gz", hash = "sha256:62c24c7fc59e42b775ce0679cfa7b14a5f9bfb7643cfbe708c960699e05fb918"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, + {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, + {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, + {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 374307890..7a59de2b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "pyodbc>=4.0.35", "python-snappy>=0.6.1", "beautifulsoup4>=4.12.2", - "dbt-bigquery>=1.5.3", + "dbt-bigquery!=1.6.5", "ldap3>=2.9.1", "pycryptodome>=3.18.0", "pandas>=2.0.3", diff --git a/src/dbt/kippmiami/models/fldoe/staging/stg_fldoe__fast.sql b/src/dbt/kippmiami/models/fldoe/staging/stg_fldoe__fast.sql index 8b5d47415..450a1d650 100644 --- a/src/dbt/kippmiami/models/fldoe/staging/stg_fldoe__fast.sql +++ b/src/dbt/kippmiami/models/fldoe/staging/stg_fldoe__fast.sql @@ -143,15 +143,31 @@ with `3_geometric_reasoning_measurement_and_data_analysis_and_probability_performance`, `4_geometric_reasoning_measurement_and_data_analysis_and_probability_performance` ) as geometric_reasoning_measurement_and_data_analysis_and_probability, - from {{ source('fldoe', 'src_fldoe__fast') }} + from {{ source("fldoe", "src_fldoe__fast") }} + ), + + with_calcs as ( + select + * except (percentile_rank), + safe_cast( + regexp_extract(percentile_rank, r'\d+') as numeric + ) as percentile_rank, + safe_cast( + regexp_extract(achievement_level, r'\d+') as int + ) as achievement_level_int, + lag(scale_score, 1) over ( + partition by student_id, academic_year, assessment_subject + order by administration_window asc + ) as scale_score_prev, + from fast_data ) select - * except (percentile_rank), - safe_cast(regexp_extract(percentile_rank, r'\d+') as numeric) as percentile_rank, - safe_cast(regexp_extract(achievement_level, r'\d+') as int) as achievement_level_int, - lag(scale_score, 1) over ( - partition by student_id, academic_year, assessment_subject - order by administration_window asc - ) as scale_score_prev, -from fast_data + *, + case + when achievement_level_int >= 3 + then true + when achievement_level_int < 3 + then false + end as is_proficient, +from with_calcs diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql index 30c3cdf60..5acc843d7 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql @@ -105,10 +105,10 @@ select dr.progress_to_stretch, dr.scale_points_to_proficiency, - ft.administration_window as pm_round, ft.achievement_level, ft.scale_score, ft.scale_score_prev, + case ft.is_proficient when true then 1.0 when false then 0.0 end as is_proficient, cwf.sublevel_name as fast_sublevel_name, cwf.sublevel_number as fast_sublevel_number, @@ -120,15 +120,18 @@ select if( not co.is_retained_year and co.grade_level = 3 - and subj.fast_subject = 'ELA' - and ft.achievement_level_int < 2, + and subj.fast_subject = 'ELAReading' + and not ft.is_proficient, 1, 0 ) as gr3_retention_flag, row_number() over ( partition by - co.fleid, co.academic_year, subj.fast_subject, ft.administration_window + co.student_number, + co.academic_year, + subj.fast_subject, + administration_window order by fs.standard asc ) as rn_test_fast, from {{ ref("base_powerschool__student_enrollments") }} as co