Pesacheck Meedan Bridge

.gitignore

@@ -165,5 +165,6 @@ cython_debug/
 ### Pyenv ###
 
 .python-version
-
+# Custom gitignore
+*.db
 # End of custom ignore

3rdparty/py/requirements-all.txt

@@ -12,6 +12,7 @@ google-auth-httplib2==0.2.0
 google-auth-oauthlib==1.2.0
 greenlet==3.0.3
 gunicorn[gevent, setproctitle]==21.2.0
+html2text==2020.1.16
 redis==5.0.1
 requests==2.31.0
 sentry-sdk==1.40.3

docker-compose.yml

@@ -48,7 +48,16 @@ services:
     depends_on:
       - twoopstracker_app
 
+  pesacheck_meedan_bridge:
+    image: codeforafrica/pesacheck_meedan_bridge:latest
+    command: ["tail", "-f", "/dev/null"]
+    volumes:
+      - pesacheck-data:/app/database
+    env_file:
+      - ./pesacheck_meedan_bridge/.env
+
 volumes:
   app-media:
   app-staticfiles:
   db-data:
+  pesacheck-data:

pants.toml

@@ -35,6 +35,8 @@ root_patterns = [
   "/twoops_tracker/docker",
   "/twoops_tracker/py",
   "/pants-plugins",
+  "/pesacheck_meedan_bridge/py",
+  "/pesacheck_meedan_bridge/docker",
 ]
 
 [python]

pesacheck_meedan_bridge/.env.example

@@ -0,0 +1,8 @@
+PESACHECK_URL=https://pesacheck.org/feed
+PESACHECK_RSS2JSON_API_KEY=
+PESACHECK_CHECK_URL=https://check-api.checkmedia.org/api/graphql
+PESACHECK_CHECK_TOKEN=
+PESACHECK_CHECK_WORKSPACE_SLUG=pesacheck-tipline-sandbox
+PESACHECK_SENTRY_DSN=
+PESACHECK_SENTRY_TRACES_SAMPLE_RATE=1.0
+PESACHECK_SENTRY_ENVIRONMENT=local

pesacheck_meedan_bridge/docker/BUILD

@@ -0,0 +1,32 @@
+docker_image(
+    name="pesacheck-deps",
+    image_tags=["deps"],
+    build_platform=["linux/amd64"],
+    registries=["pesacheck_meedan_bridge"],
+    repository="app",
+    skip_push=True,
+    source="Dockerfile.deps",
+)
+
+file(name="app.json", source="app.json")
+
+docker_image(
+    name="pesacheck-srcs",
+    image_tags=["srcs"],
+    build_platform=["linux/amd64"],
+    registries=["pesacheck_meedan_bridge"],
+    repository="app",
+    skip_push=True,
+    source="Dockerfile.srcs",
+)
+
+docker_image(
+    name="pesacheck_meedan_bridge",
+    build_platform=["linux/amd64"],
+    dependencies=[":pesacheck-srcs", ":pesacheck-deps", ":app.json"],
+    image_tags=[
+        "{build_args.VERSION}",
+        "latest",
+    ],
+    source="Dockerfile",
+)

pesacheck_meedan_bridge/docker/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.11-slim-bullseye AS python-base
+FROM pesacheck_meedan_bridge/app:deps AS app-deps
+FROM pesacheck_meedan_bridge/app:srcs AS app-srcs
+FROM python-base AS python-app
+
+WORKDIR /app
+COPY pesacheck_meedan_bridge/docker/app.json ./
+COPY --from=app-deps /app ./
+COPY --from=app-srcs /app ./
+
+CMD ["tail", "-f", "/dev/null"]

pesacheck_meedan_bridge/docker/Dockerfile.deps

@@ -0,0 +1,4 @@
+FROM python:3.11-slim-bookworm
+
+COPY pesacheck_meedan_bridge.py/pesacheck-deps@environment=linux.pex /pesacheck-deps.pex
+RUN PEX_TOOLS=1 python /pesacheck-deps.pex venv --scope=deps --compile /app

pesacheck_meedan_bridge/docker/Dockerfile.srcs

@@ -0,0 +1,4 @@
+FROM python:3.11-slim-bookworm
+
+COPY pesacheck_meedan_bridge.py/pesacheck-srcs@environment=linux.pex /pesacheck-srcs.pex
+RUN PEX_TOOLS=1 python /pesacheck-srcs.pex venv --scope=srcs --compile /app

pesacheck_meedan_bridge/docker/app.json

@@ -0,0 +1,9 @@
+{
+    "name": "pesacheck_meedan_bridge",
+    "cron": [
+        {
+            "command": "./pex",
+            "schedule": "@daily"
+        }
+    ]
+}

pesacheck_meedan_bridge/py/BUILD

@@ -0,0 +1,42 @@
+python_sources(
+    name="lib",
+    dependencies=[
+        "3rdparty/py:requirements-all#requests",
+        "3rdparty/py:requirements-all#sentry-sdk",
+        "3rdparty/py:requirements-all#environs",
+        "3rdparty/py:requirements-all#html2text",
+    ],
+)
+
+pex_binary(
+    name="pesacheck-deps",
+    environment=parametrize("__local__", "linux"),
+    dependencies=[
+        ":lib",
+    ],
+    entry_point="main.py",
+    include_sources=False,
+    include_tools=True,
+    layout="packed",
+)
+
+pex_binary(
+    name="pesacheck-srcs",
+    environment=parametrize("__local__", "linux"),
+    dependencies=[
+        ":lib",
+    ],
+    entry_point="main.py",
+    include_requirements=False,
+    include_tools=True,
+    layout="packed",
+)
+
+
+pex_binary(
+    name="pesacheck_meedan_bridge",
+    dependencies=[
+        ":lib",
+    ],
+    entry_point="main.py",
+)

pesacheck_meedan_bridge/py/VERSION

@@ -0,0 +1 @@
+0.1.0

pesacheck_meedan_bridge/py/check_api.py

@@ -0,0 +1,77 @@
+import json
+
+import requests
+import settings
+
+
+def create_mutation_query(
+    media_type="Blank",
+    channel=None,
+    set_tags=[],
+    set_status="",
+    set_claim_description="",
+    title="",
+    summary="",
+    url="",
+    language="",
+    publish_report=False,
+):
+    mutation_query = f"""
+      mutation create {{
+        createProjectMedia(input: {{
+          media_type: "{media_type}",
+          channel: {{ main: {channel} }},
+          set_tags: {json.dumps(set_tags)},
+          set_status: "{set_status}",
+          set_claim_description: \"\"\"{set_claim_description}\"\"\",
+          set_fact_check: {{
+            title: \"\"\"{title}\"\"\",
+            summary: \"\"\"{summary}\"\"\",
+            url: "{url}",
+            language: "{language}",
+            publish_report: {str(publish_report).lower()}
+          }}
+        }}) {{
+          project_media {{
+            id
+            full_url
+            claim_description {{
+              fact_check {{
+                id
+              }}
+            }}
+          }}
+        }}
+      }}
+    """
+
+    return mutation_query
+
+
+def delete_mutation_query(id):
+    mutation_query = f"""
+    mutation {{
+      destroyFactCheck(input: {{
+        id: "{id}"
+      }}) {{ deletedId }}
+    }}
+    """
+    return mutation_query
+
+
+def post_to_check(data):
+    query = create_mutation_query(**data)
+    headers = {
+        "Content-Type": "application/json",
+        "X-Check-Token": settings.PESACHECK_CHECK_TOKEN,
+        "X-Check-Team": settings.PESACHECK_CHECK_WORKSPACE_SLUG,
+    }
+    body = dict(query=query)
+    url = settings.PESACHECK_CHECK_URL
+    response = requests.post(url, headers=headers, json=body, timeout=60)
+    if response.status_code == 200:
+        res = response.json()
+        if res.get("errors"):
+            raise Exception(res["errors"])
+        return res
+    raise Exception(response.text)

pesacheck_meedan_bridge/py/database.py

@@ -0,0 +1,139 @@
+import sqlite3
+from dataclasses import dataclass
+from sqlite3 import Error
+
+import sentry_sdk
+import settings
+
+
+@dataclass
+class PesacheckFeed:
+    title: str
+    pubDate: str
+    author: str
+    guid: str
+    link: str
+    thumbnail: str
+    description: str
+    status: str
+    categories: str
+    check_project_media_id: str = ""
+    check_full_url: str = ""
+    claim_description_id: str = ""
+
+
+class PesacheckDatabase:
+    def __init__(self):
+        self.db_file = settings.PESACHECK_DATABASE_NAME
+        self.create_table()
+
+    def create_connection(self):
+        try:
+            conn = sqlite3.connect(self.db_file)
+            return conn
+        except Error as e:
+            sentry_sdk.capture_exception(e)
+        return None
+
+    def create_table(self):
+        conn = self.create_connection()
+        try:
+            cursor = conn.cursor()
+            cursor.execute(
+                """CREATE TABLE IF NOT EXISTS pesacheck_feeds
+                              (title TEXT NOT NULL,
+                              pubDate TEXT NOT NULL,
+                              author TEXT NOT NULL,
+                              guid TEXT PRIMARY KEY,
+                              link TEXT NOT NULL,
+                              thumbnail TEXT NOT NULL,
+                              description TEXT NOT NULL,
+                              status TEXT DEFAULT 'Pending',
+                              categories TEXT DEFAULT '[]',
+                              check_project_media_id TEXT,
+                              check_full_url TEXT,
+                              claim_description_id TEXT)"""
+            )
+            conn.commit()
+        except Error as e:
+            sentry_sdk.capture_exception(e)
+        finally:
+            conn.close()
+
+    def insert_pesacheck_feed(self, feed):
+        conn = self.create_connection()
+        sql = """INSERT INTO pesacheck_feeds (title, pubDate, author,
+                 guid, link, thumbnail, description, status, categories,
+                 check_project_media_id, check_full_url, claim_description_id)
+                 VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+        try:
+            cur = conn.cursor()
+            cur.execute(
+                sql,
+                (
+                    feed.title,
+                    feed.pubDate,
+                    feed.author,
+                    feed.guid,
+                    feed.link,
+                    feed.thumbnail,
+                    feed.description,
+                    feed.status,
+                    feed.categories,
+                    feed.check_project_media_id,
+                    feed.check_full_url,
+                    feed.claim_description_id,
+                ),
+            )
+            conn.commit()
+        except Error as e:
+            sentry_sdk.capture_exception(e)
+        finally:
+            conn.close()
+
+    def update_pesacheck_feed(self, guid, new_feed):
+        conn = self.create_connection()
+        sql = """UPDATE pesacheck_feeds
+                SET title = ?, pubDate = ?, author = ?, link = ?, thumbnail = ?,
+                description = ?, status = ?, categories = ?,
+                check_project_media_id = ?, check_full_url = ?,
+                claim_description_id = ? WHERE guid = ?"""
+        try:
+            cur = conn.cursor()
+            cur.execute(
+                sql,
+                (
+                    new_feed.title,
+                    new_feed.pubDate,
+                    new_feed.author,
+                    new_feed.link,
+                    new_feed.thumbnail,
+                    new_feed.description,
+                    new_feed.status,
+                    new_feed.categories,
+                    new_feed.check_project_media_id,
+                    new_feed.check_full_url,
+                    new_feed.claim_description_id,
+                    guid,
+                ),
+            )
+            conn.commit()
+        except Error as e:
+            sentry_sdk.capture_exception(e)
+        finally:
+            conn.close()
+
+    def get_pending_pesacheck_feeds(self):
+        conn = self.create_connection()
+        try:
+            cur = conn.cursor()
+            cur.execute("SELECT * FROM pesacheck_feeds WHERE status = 'Pending'")
+            rows = cur.fetchall()
+            feeds = []
+            for row in rows:
+                feeds.append(PesacheckFeed(*row))
+            return feeds
+        except Error as e:
+            sentry_sdk.capture_exception(e)
+        finally:
+            conn.close()