Update to latest data repo config

mysociety · Sep 7, 2023 · 348810f · 348810f
1 parent c019931
commit 348810f
Show file tree

Hide file tree

Showing 8 changed files with 1,926 additions and 1,686 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,6 @@ _render/_parts
 _render/_papermills
 docs/_site
 data/raw/scrapedxml
-data/interim/regmem/
+data/interim/regmem/
+docs/data
+docs/_*
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,15 +1,14 @@
 {
-    "sqlfluff.dialect": "ansi",
-    "sqlfluff.format.arguments": [
-        "--ignore=templating"
-    ],
-    "sqlfluff.linter.arguments": [
-        "--ignore=templating"
-    ],
-    "python.linting.pylintEnabled": true,
+    "[python]": {
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.fixAll.ruff": true,
+            "source.organizeImports.ruff": true
+        },
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
     "python.defaultInterpreterPath": "/usr/local/bin/python",
     "python.terminal.activateEnvironment": false,
-    "python.formatting.provider": "black",
     "python.analysis.typeCheckingMode": "basic",
     "python.analysis.stubPath": "src/data_common/typing",
     "editor.formatOnSave": true,
@@ -30,12 +29,6 @@
         "**/templates/**/*": "django-txt",
         "**/requirements{/**,*}.{txt,in}": "pip-requirements"
     },
-    "python.linting.pylintArgs": [
-        "--max-line-length=88",
-        "--disable=C0103,E1101,W5101,E1123,E501,E203",
-        "--load-plugins=pylint_django"
-    ],
-    "jupyter.jupyterServerType": "local",
     "ltex.language": "en-GB",
     "ltex.ltex-ls.path": "/ltex/ltex-ls-15.2.0/",
     "[markdown]": {

diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 # This dockerfile is used by binder.
 
-FROM ghcr.io/mysociety/data_common:sha-2d20c80
+FROM ghcr.io/mysociety/data_common:sha-a3b8e93
 
 # Make an empty project directory so the 'self' setup doesn't fail and scripts can be setup
 # Override the .pth created at previous stages to point to where the working directory will land

diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM ghcr.io/mysociety/data_common:sha-2d20c80
+FROM ghcr.io/mysociety/data_common:sha-a3b8e93
 
 # Make an empty project directory so the 'self' setup doesn't fail and scripts can be setup
 # Override the .pth created at previous stages to point to where the working directory will land

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -24,7 +24,6 @@ exclude=["src/data_common/typings"]
 stubPath="src/data_common/typings"
 typeCheckingMode="basic"
 
-
 [notebook.settings]
 default_page_title = "Settings file defined title"
 
@@ -37,3 +36,7 @@ credit_url = "https://survey.alchemer.com/s3/6876792/Data-usage"
 
 [tool.duck]
 query_dir = "src/parl_register_interests/sql"
+
+[tool.ruff]
+select = ["E", "F"]
+ignore = ["E501"]
diff --git a/script/setup b/script/setup
@@ -1,3 +1,3 @@
 #!/bin/bash 
 apt-get install rsync -y
-python -m spacy download en_core_web_lg
+python -m spacy download en_core_web_md
diff --git a/src/data_common b/src/data_common
+4 −1		bin/packages_setup.bash
+1,799 −1,267		poetry.lock
+2 −1		pyproject.toml
+2 −2		src/data_common/charting/download.py
+1 −1		src/data_common/charting/sw_theme.py
+1 −1		src/data_common/charting/theme.py
+1 −4		src/data_common/dataset/jekyll_management.py
+75 −35		src/data_common/dataset/resource_management.py
+4 −1		src/data_common/dataset/table_management.py
+109 −9		src/data_common/db/duck.py
+104 −0		src/data_common/helpers/parquet.py
+2 −2		src/data_common/helpers/pipe.py
+119 −0		src/data_common/helpers/typing.py
+204 −0		src/data_common/helpers/url.py
+5 −3		src/data_common/management/run_notebook.py
+5 −1		src/data_common/management/settings.py
+1 −7		src/data_common/pandas/df_extensions/space.py
+3 −0		src/data_common/resources/dataset_template/{{cookiecutter.slug}}/datapackage.yaml