chore: verify data availability using cronjob (#232)

* chore: verify data availability via cronjob instead of in every PR

* fix some formatting issue

.github/workflows/data.yml

@@ -0,0 +1,23 @@
+name: Tests if open data is properly accessible
+
+on:
+  schedule:
+  - cron: "0 10 * * 0" # every sunday 10am
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: "ile-de-france"
+        environment-file: environment.yml
+        python-version: "3.10"
+        channels: conda-forge
+
+    - name: Verify availability of open data
+      shell: bash -el {0}
+      run: |
+        python scripts/verify_data.py

.github/workflows/tests.yml

@@ -70,9 +70,3 @@ jobs:
       shell: bash -el {0}
       run: |
         MKL_CBWR=AUTO pytest tests/
-    - name: Verify availability of open data
-      if: matrix.os == 'ubuntu-latest'
-      shell: bash -el {0}
-      run: |
-        python scripts/verify_data.py
-

docs/verify_data.py

@@ -80,7 +80,7 @@
     },
 
     {
-        "name": "SIRET géolocalisé",
+        "name": "SIRET géolocalisé",
         "urls": [
             "https://adresse.data.gouv.fr/donnees-nationales"
         ]
@@ -121,12 +121,15 @@
 ]
 
 # Start testing process
+import time
 from urllib.request import urlopen
 
 any_errors = False
+sleep_time = 10 # s
 
 for test in tests:
     print("Testing %s ..." % test["name"])
+    time.sleep(sleep_time)
 
     for url in test["urls"]:
         try: