Pesacheck Meedan Bridge

.dockerignore

@@ -0,0 +1 @@
+**/venv

.gitignore

@@ -165,3 +165,4 @@ cython_debug/
 /.pants.workdir.file_lock*
 
 # End of custom ignore
+credentials.json

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.linting.enabled": true,
+    "cSpell.words": [
+        "dotenv"
+    ]
+}

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.9
+
+WORKDIR /app
+COPY . /app
+
+RUN pip install virtualenv
+ENV PATH="/app/pesacheck_meedan_bridge/venv/bin:$PATH"
+RUN virtualenv /app/pesacheck_meedan_bridge/venv
+RUN echo "source /app/pesacheck_meedan_bridge/venv/bin/activate" >> ~/.bashrc
+RUN pip install --no-cache-dir -r pesacheck_meedan_bridge/requirements.txt
+
+RUN apt-get update && apt-get -y install cron
+
+RUN chmod +x /app/pesacheck_meedan_bridge/service.py
+ADD crontab /etc/cron.d/crontab
+
+RUN chmod 0644 /etc/cron.d/crontab
+
+RUN crontab /etc/cron.d/crontab
+
+CMD ["cron", "-f"]
+

app.json

@@ -0,0 +1,8 @@
+{
+    "cron": [
+        {
+            "command": "./test_script.sh",
+            "schedule": "* * * * *"
+        }
+    ]
+}

crontab

@@ -0,0 +1 @@
+0 0 * * * root /app/pesacheck_meedan_bridge/venv/bin/python /app/pesacheck_meedan_bridge/service.py && echo "Cron job ran at $(date)" >> /app/debug.log

docker-compose.yml

@@ -46,7 +46,10 @@ services:
       - 8000:80
     depends_on:
       - twoopstracker_app
-
+  pesacheck_meedan_bridge:
+    build:
+      context: .
+      dockerfile: Dockerfile
 volumes:
   app-media:
   app-staticfiles:

pesacheck_meedan_bridge/.env.example

@@ -0,0 +1,6 @@
+PESACHECK_URL=https://pesacheck.org/feed
+PESACHECK_RSS_2_JSON_API_KEY=
+PESACHECK_CHECK_URL=https://check-api.checkmedia.org/api/graphql
+PESACHECK_TOKEN=
+PESACHECK_WORKSPACE_SLUG=pesacheck-tipline-sandbox
+PESACHECK_SENTRY_DSN=

pesacheck_meedan_bridge/mutation_queries.py

@@ -0,0 +1,48 @@
+import json
+
+
+def create_mutation_query(media_type=None, channel=None, set_tags=None,
+                          set_status=None, set_claim_description=None,
+                          title=None, summary=None, url=None,
+                          language=None, publish_report=False):
+    mutation_query = f"""
+      mutation create {{
+        createProjectMedia(input: {{
+          media_type: "{media_type or "Blank"}",
+          channel: {{ main: {channel} }},
+          set_tags: {json.dumps(set_tags or [])},
+          set_status: "{set_status or ""}",
+          set_claim_description: "{set_claim_description or ""}",
+          set_fact_check: {{
+            title: "{title or ""}",
+            summary: "{summary or ""}",
+            url: "{url or ""}",
+            language: "{language or ""}",
+            publish_report: {str(publish_report).lower()}
+          }}
+        }}) {{
+          project_media {{
+            id
+            full_url
+            claim_description {{
+              fact_check {{
+                id
+              }}
+            }}
+          }}
+        }}
+      }}
+    """
+
+    return mutation_query
+
+
+def delete_mutation_query(id):
+    mutation_query = f'''
+    mutation {{
+      destroyFactCheck(input: {{
+        id: "{id}"
+      }}) {{ deletedId }}
+    }}
+    '''
+    return mutation_query

pesacheck_meedan_bridge/output.json

@@ -0,0 +1,152 @@
+[
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyMzg=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637238",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIxNA=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyMzk=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637239",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIxNQ=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDA=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637240",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIxNg=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDE=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637241",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIxNw=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDI=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637242",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIxOA=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDM=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637243",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIxOQ=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDQ=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637244",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIyMA=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDU=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637245",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIyMQ=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDY=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637246",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIyMg=="
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        "data": {
+            "createProjectMedia": {
+                "project_media": {
+                    "id": "UHJvamVjdE1lZGlhLzI2MzcyNDc=",
+                    "full_url": "https://checkmedia.org/pesacheck-tipline-sandbox/project/16148/media/2637247",
+                    "claim_description": {
+                        "fact_check": {
+                            "id": "RmFjdENoZWNrLzgyNzIyMw=="
+                        }
+                    }
+                }
+            }
+        }
+    }
+]

pesacheck_meedan_bridge/requirements.txt

@@ -0,0 +1,7 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
+idna==3.6
+python-dotenv==1.0.1
+requests==2.31.0
+sentry-sdk==1.40.0
+urllib3==2.2.0

pesacheck_meedan_bridge/service.py

@@ -0,0 +1,76 @@
+import re
+import requests
+import os
+from dotenv import load_dotenv
+from mutation_queries import create_mutation_query, delete_mutation_query
+from sentry_sdk import init, capture_exception, capture_message, set_context
+
+load_dotenv()
+
+
+init(
+    dsn=os.getenv("PESACHECK_SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    profiles_sample_rate=1.0,
+)
+
+
+def log_error(code, message=""):
+    capture_message(f"Status code: {code}\n Error: {message}")
+
+
+def remove_html_tags(input_string):
+    return re.sub(r'<[^>]*>', '', input_string)
+
+
+def fetch_from_pesacheck():
+    url = "https://api.rss2json.com/v1/api.json"
+    rss_api_key = os.getenv("PESACHECK_RSS_2_JSON_API_KEY")
+    feed_url = os.getenv("PESACHECK_URL")
+    params = {'rss_url': feed_url, 'api_key': rss_api_key, 'count': 10, 'order_by': 'pubDate'}
+    response = requests.get(url, params=params, timeout=60)
+    if response.status_code == 200:
+        return response.json().get('items') or []
+    log_error(code=response.status_code, message=response.text)
+    return []
+
+
+def post_to_check(query):
+    try:
+        headers = {"Content-Type": "application/json",
+                   "X-Check-Token": os.getenv("PESACHECK_TOKEN"), "X-Check-Team": os.getenv("PESACHECK_WORKSPACE_SLUG")}
+        body = dict(query=query)
+        url = os.getenv("PESACHECK_CHECK_URL")
+        response = requests.post(url, headers=headers, json=body, timeout=60)
+        if response.status_code == 200:
+            return response.json()
+        log_error(code=response.status_code, message=response.text)
+        return None
+    except Exception as e:
+        capture_exception(e)
+
+
+def upload_content():
+    items = fetch_from_pesacheck()
+    success_posts = []
+    for _, item in enumerate(items):
+        language = "fr" if "french" in item.get("categories") else "en"
+        input_data = {
+            "media_type": "Blank",
+            "channel": 1,
+            "set_tags": item["categories"],
+            "set_status": "verified",
+            "set_claim_description": remove_html_tags(item["description"]),
+            "title": remove_html_tags(item["title"]),
+            "summary": remove_html_tags(item["description"]),
+            "url": item["link"],
+            "language": language,
+            "publish_report": True,
+        }
+        query = create_mutation_query(**input_data)
+        res = post_to_check(query)
+        if (res):
+            success_posts.append(res)
+    set_context("Successful Uploads", success_posts)
+
+upload_content()