From 5805026519754b87e282537f1290e6313c7a3d7c Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Fri, 22 Nov 2024 11:52:46 +0300 Subject: [PATCH 1/7] Yaml files should preferably have .yaml extensions https://yaml.org/faq.html --- .../workflows/{twoops-tracker-cd.yml => twoops-tracker-cd.yaml} | 0 .../workflows/{twoops-tracker-ci.yml => twoops-tracker-ci.yaml} | 0 docker-compose.yml => compose.yaml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{twoops-tracker-cd.yml => twoops-tracker-cd.yaml} (100%) rename .github/workflows/{twoops-tracker-ci.yml => twoops-tracker-ci.yaml} (100%) rename docker-compose.yml => compose.yaml (100%) diff --git a/.github/workflows/twoops-tracker-cd.yml b/.github/workflows/twoops-tracker-cd.yaml similarity index 100% rename from .github/workflows/twoops-tracker-cd.yml rename to .github/workflows/twoops-tracker-cd.yaml diff --git a/.github/workflows/twoops-tracker-ci.yml b/.github/workflows/twoops-tracker-ci.yaml similarity index 100% rename from .github/workflows/twoops-tracker-ci.yml rename to .github/workflows/twoops-tracker-ci.yaml diff --git a/docker-compose.yml b/compose.yaml similarity index 100% rename from docker-compose.yml rename to compose.yaml From 63ec6613a564ebac20c4e53365490f8bf6345849 Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Fri, 22 Nov 2024 11:53:58 +0300 Subject: [PATCH 2/7] Add the usual python project lint configs --- .editorconfig | 19 +++++++++++++++++++ .flake8 | 3 +++ .yamllint.yaml | 9 +++++++++ pyproject.toml | 6 ++++++ 4 files changed, 37 insertions(+) create mode 100644 .editorconfig create mode 100644 .flake8 create mode 100644 .yamllint.yaml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..df8ced3a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +# https://editorconfig.org/ + +root = true + +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf +charset = utf-8 + +# Docstrings and comments use max_line_length = 79 +[*.py] +profile = black + +# Makefiles always use tabs for indentation +[Makefile] +indent_style = tab diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..6c032f36 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203,E701 \ No newline at end of file diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 00000000..742192b9 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,9 @@ +extends: default + +rules: + # https://yamllint.readthedocs.io/en/stable/rules.html#module-yamllint.rules.line_length + line-length: + max: 88 # profile=black + # https://yamllint.readthedocs.io/en/stable/rules.html#module-yamllint.rules.document_start + document-start: + present: false diff --git a/pyproject.toml b/pyproject.toml index 21c3b15f..8131b28c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ +[project] +requires-python = ">= 3.11" + [tool.isort] profile = "black" @@ -5,5 +8,8 @@ profile = "black" ignore_missing_imports = true # TODO(kilemensi): mypy django stubs +[tool.ruff] +target-version = "py311" # default: py39 + [tool.setuptools_scm] write_to = "VERSION" From 296656778459fa53e34f0a7f800e371ac70e6dab Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Fri, 22 Nov 2024 11:55:17 +0300 Subject: [PATCH 3/7] Upgrade pants to 2.23 and switch to ruff --- pants.toml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/pants.toml b/pants.toml index 42099dae..8379a011 100644 --- a/pants.toml +++ b/pants.toml @@ -1,14 +1,15 @@ [GLOBAL] -pants_version = "2.19.0" +pants_version = "2.23.0" pythonpath = ["%(buildroot)s/pants-plugins"] backend_packages.add = [ - "pants.backend.build_files.fmt.black", + "pants.backend.build_files.fmt.ruff", "pants.backend.docker", "pants.backend.docker.lint.hadolint", "pants.backend.plugin_development", "pants.backend.python", - "pants.backend.python.lint.black", + "pants.backend.python.mixed_interpreter_constraints", + "pants.backend.python.lint.bandit", "pants.backend.python.lint.docformatter", "pants.backend.python.lint.flake8", "pants.backend.python.lint.isort", @@ -21,6 +22,8 @@ backend_packages.add = [ # Expiremental backends "pants.backend.experimental.python", "pants.backend.experimental.python.framework.django", + "pants.backend.experimental.python.lint.ruff.check", + "pants.backend.experimental.python.lint.ruff.format", "pants.backend.experimental.tools.yamllint", # In-repo plugins "internal_plugins.versioned_setup", @@ -38,10 +41,24 @@ root_patterns = [ ] [python] +enable_resolves = false interpreter_constraints = ["==3.11.*"] -[black] -args = ["--preview"] +[python-repos] +find_links = ["https://wheels.pantsbuild.org/simple"] + +[generate-lockfiles] +diff = true + +[bandit] +args = ["--quiet"] + +[mypy] +requirements = ["django-stubs"] +interpreter_constraints = ["==3.11.*"] + +[pyupgrade] +args = ["--py36-plus"] [docker] build_args = ["VERSION"] From 309e81dba2bf8fc5b9428defdd032acfb1c72d34 Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Fri, 22 Nov 2024 11:56:23 +0300 Subject: [PATCH 4/7] Lint yaml files --- .github/workflows/twoops-tracker-cd.yaml | 3 ++- .github/workflows/twoops-tracker-ci.yaml | 4 +++- compose.yaml | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/twoops-tracker-cd.yaml b/.github/workflows/twoops-tracker-cd.yaml index da47076d..deee86d3 100644 --- a/.github/workflows/twoops-tracker-cd.yaml +++ b/.github/workflows/twoops-tracker-cd.yaml @@ -1,5 +1,6 @@ +# yamllint disable rule:line-length name: TwoopsTracker | Continuous Deployment -on: +"on": push: branches: - main diff --git a/.github/workflows/twoops-tracker-ci.yaml b/.github/workflows/twoops-tracker-ci.yaml index e99db8c3..c4cd3dfb 100644 --- a/.github/workflows/twoops-tracker-ci.yaml +++ b/.github/workflows/twoops-tracker-ci.yaml @@ -1,10 +1,12 @@ +# yamllint disable rule:line-length name: TwoopsTracker | Continuous Integration -on: +"on": pull_request: types: [opened, synchronize, reopened] # This allows a subsequently queued workflow run to interrupt previous runs concurrency: + # yamllint disable-line rule:line-length group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" cancel-in-progress: true diff --git a/compose.yaml b/compose.yaml index 00c74fc3..43ce7add 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,3 @@ ---- services: db: image: postgres:13.4 From 070779589aec394332b3cabef37e98ad4320c3d0 Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Fri, 22 Nov 2024 11:56:42 +0300 Subject: [PATCH 5/7] Fix dependencies --- 3rdparty/py/requirements-all.txt | 5 ++++- 3rdparty/py/requirements-dev.txt | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/3rdparty/py/requirements-all.txt b/3rdparty/py/requirements-all.txt index 95e93c83..4f1041b5 100644 --- a/3rdparty/py/requirements-all.txt +++ b/3rdparty/py/requirements-all.txt @@ -1,6 +1,8 @@ boto3==1.34.39 celery==5.3.6 -dj-rest-auth[with_social]==5.0.2 +dj-rest-auth==5.0.2 +# Match dj-rest-auth[with_social]==5.0.2 +django-allauth>=0.56.0,<0.58.0 Django==3.2.23 django-cors-headers==4.3.1 django-storages==1.14.2 @@ -8,6 +10,7 @@ djangorestframework==3.14.0 djangorestframework-simplejwt==5.3.1 environs[django]==10.3.0 google-api-python-client==2.117.0 +google-auth==2.36.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.0 greenlet==3.0.3 diff --git a/3rdparty/py/requirements-dev.txt b/3rdparty/py/requirements-dev.txt index f7b8d7f3..bcc7a641 100644 --- a/3rdparty/py/requirements-dev.txt +++ b/3rdparty/py/requirements-dev.txt @@ -1,4 +1,3 @@ -r requirements-all.txt -black==24.1.1 psycopg2-binary==2.9.9 From 7021e8fc348a8d1e533d6195b9d3b09dbf9e50a7 Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Fri, 22 Nov 2024 11:57:00 +0300 Subject: [PATCH 6/7] Lint --- twoops_tracker/py/BUILD | 2 ++ twoops_tracker/py/twoopstracker/urls.py | 1 + 2 files changed, 3 insertions(+) diff --git a/twoops_tracker/py/BUILD b/twoops_tracker/py/BUILD index 52296685..ae7c1515 100644 --- a/twoops_tracker/py/BUILD +++ b/twoops_tracker/py/BUILD @@ -6,9 +6,11 @@ python_sources( "3rdparty/py:requirements-all#django-storages", "3rdparty/py:requirements-all#djangorestframework-simplejwt", "3rdparty/py:requirements-all#google-api-python-client", + "3rdparty/py:requirements-all#google-auth", "3rdparty/py:requirements-all#google-auth-httplib2", "3rdparty/py:requirements-all#google-auth-oauthlib", "3rdparty/py:requirements-all#dj-rest-auth", + "3rdparty/py:requirements-all#django-allauth", "3rdparty/py:requirements-all#gunicorn", "3rdparty/py:requirements-all#celery", "3rdparty/py:requirements-all#redis", diff --git a/twoops_tracker/py/twoopstracker/urls.py b/twoops_tracker/py/twoopstracker/urls.py index b48a62f6..56baa7fe 100644 --- a/twoops_tracker/py/twoopstracker/urls.py +++ b/twoops_tracker/py/twoopstracker/urls.py @@ -1,4 +1,5 @@ """twoopstracker URL Configuration.""" + from django.contrib import admin from django.urls import include, path From 627c3dccedd9eabd21f391263f42e40f26daadf4 Mon Sep 17 00:00:00 2001 From: Clemence Kyara Date: Fri, 22 Nov 2024 11:57:16 +0300 Subject: [PATCH 7/7] Update docs --- README.md | 10 ++++++++++ twoops_tracker/README.md | 16 ++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 89f2f21c..03c9d26d 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,13 @@ | Name | Description | | ---------------------------------- | -------------------------------------------------------------- | | [**`dj-cmd`**](./dj-cmd/README.md) | Command-line utility for running Django when bundled in `pex`. | + +## Development + +This [`monorepo`](https://monorepo.tools/) uses [`Pantsbuild`](https://www.pantsbuild.org/) build system. To get started, please ensure you have [`Pants`](https://www.pantsbuild.org/stable/docs/getting-started/installing-pants) installed on your machine. + +### Pants Plugins + +| Name | Description | +| ----------------------------------------------- | -------------------------------------------- | +| **VersionSetup** | | diff --git a/twoops_tracker/README.md b/twoops_tracker/README.md index 45bbcdf8..7bc0a18f 100644 --- a/twoops_tracker/README.md +++ b/twoops_tracker/README.md @@ -1,4 +1,4 @@ -## TwoopsTracker +# TwoopsTracker A Twitter-based disinformation tracking tool built on a web-based dashboard that collects deleted tweet content from previously identified trolls and disinformation actors. The project seeks to help monitor the social posts of known disinfo actors. The primary tangible output of the project is to expose trolls behind toxic disinformation campaigns who routinely cover their tracks by deleting original inflammatory social media posts that sparked hate speech, disinformation campaigns or conspiracy theories. @@ -14,30 +14,30 @@ and modify the `.env` file according to your needs. ## Build -To build a pex binary, run: +To build a `pex` binary, run: ```sh -./pants package twoops_tracker/py:twoopstracker +pants package twoops_tracker/py:twoopstracker ``` To build the docker image, run: ```sh -VERSION=$(cat twoops_tracker/py/VERSION) ./pants package twoops_tracker/docker:twoopstracker +VERSION=$(cat twoops_tracker/py/VERSION) pants package twoops_tracker/docker:twoopstracker ``` ## Run -To run pex binary, execute: +To run `pex` binary, execute: ```sh -./pants run twoops_tracker/py:twoopstracker +pants run twoops_tracker/py:twoopstracker ``` To run the built docker image, execute: ```sh -docker-compose --env-file ./twoops_tracker/.env up twoopstracker +docker compose --env-file ./twoops_tracker/.env up twoopstracker ``` **NOTE**: You may need to run `postres` container first to make sure database @@ -46,5 +46,5 @@ is ready to receive connections _before_ starting the `twoopstracker` app. To do so, run: ```sh -docker-compose --env-file ./twoops_tracker/.env up db +docker compose --env-file ./twoops_tracker/.env up db ```