Merge pull request #927 from openzim/upgrade_uploader

Upgrade uploader

dev/README.md

@@ -29,6 +29,10 @@ This container is a PostgreSQL DB. DB data is kept in a volume, persistent acros
 
 This container hosts the frontend UI for end-users.
 
+### ssh-host
+
+This container hosts a minimal SSH server, usefull for uploading Zimfarm artifacts locally during tests.
+
 ## Instructions
 
 First start the Docker-Compose stack:

dev/docker-compose.yml

@@ -28,6 +28,9 @@ services:
       JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw
       POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@postgresdb:5432/zimfarm
       ALEMBIC_UPGRADE_HEAD_ON_START: "1"
+      ARTIFACTS_UPLOAD_URI: scp://root@ssh-host:22/artifacts/
+      LOGS_UPLOAD_URI: scp://root@ssh-host:22/logs/
+      ZIM_UPLOAD_URI: scp://root@ssh-host:22/zims/
     depends_on:
       - postgresdb
   frontend-ui:
@@ -78,6 +81,13 @@ services:
       POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@postgresdb:5432/zimtest
     depends_on:
       - postgresdb
+  ssh-host:
+    build:
+      context: ssh-host
+    container_name: zf_ssh_host
+    ports:
+      - 127.0.0.1:8022:22
+
 
 volumes:
   pg_data_zimfarm:

dev/ssh-host/Dockerfile

@@ -0,0 +1,44 @@
+FROM alpine:3
+
+# Install SSH server
+RUN apk update && apk add --no-cache openssh-server
+
+# Create SSH directory and set permissions
+RUN mkdir /var/run/sshd
+RUN chmod 0755 /var/run/sshd
+
+# Copy SSH host keys to container
+COPY ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key
+
+# Copy test client SSH public key to the container
+COPY id_rsa.pub /root/.ssh/authorized_keys
+
+RUN mkdir -p \
+    /root/artifacts \
+    /root/logs \
+    /root/zims/freecodecamp \
+    /root/zims/gutenberg \
+    /root/zims/ifixit \
+    /root/zims/mooc \
+    /root/zims/other \
+    /root/zims/phet \
+    /root/zims/stack_exchange \
+    /root/zims/ted \
+    /root/zims/videos \
+    /root/zims/vikidia \
+    /root/zims/wikibooks \
+    /root/zims/wikihow \
+    /root/zims/wikinews \
+    /root/zims/wikipedia \
+    /root/zims/wikiquote \
+    /root/zims/wikisource \
+    /root/zims/wikiversity \
+    /root/zims/wikivoyage \
+    /root/zims/wiktionary \
+    /root/zims/zimit
+
+# Expose SSH port
+EXPOSE 22
+
+# Start SSH server
+CMD ["/usr/sbin/sshd", "-D"]

dev/ssh-host/id_rsa

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAp9q+SGUiadNhQQGEmQUEXTgSgStlThK3awQmZuSIN4QykLsa
+t/PluQidiVCVPi0Jv/7np5ufLHBpRLZOfsOUozm48yvEMeiXkYQImfMacsnUqFCL
+3dAEQ/TisHYkBalZSz5Tsq3SKUNr0REncM4XtH+xGFMICswcOZNH6MULBsU4s7WP
+7oFifCbKOVjsV8MxMOHEpxoXa3oai6YZWWp6+eMI+aE3K4/7yR5aW8ZGMxTHNmYI
+/NRnSFeMj9x9cl+MkivFZAzzgnMx2nHUZx8OLvFYH+sEumct4jKb7hruwSZMDaE1
+75lVeQV1JuhgDEkEwD3MOU/DPnl40gJ64hgwCwIDAQABAoIBAAQzoQy931knzejr
+KU8NRZkxxKDQHcaRCF9Y6L8tnE1LdVqTHG2bYNY7ZXog9sNqLDLOpizXY+ogXDo4
+mAlrSua1FLCdmVkoedbOJ8r2v2Q9MiIOdhnU0a6DzELfijJMFWyhQEYaPMcEpYfD
+tqmHtycxBRVXSr2+czxrt+KXQBnovoqkM3M/ritAyj6HhQ8ZgqecmKA0UNWHXMX9
+fYJqeHtoek7DOM30C2qUOfiTE8Cd92BFyFffajL13tZPS0hD1yN3rE11khCbMfMo
+5n4lKykBw4G6EX4+mzTLPNxK29sead9IYfa0RteJIVXZG4GX7wnok8vVLm2f6k0N
+iMsNPMUCgYEA1t2X/7y/x/RMOpZ3dDvK2Zm3bnICah7nXK7afs/WFUiwkzFg92S1
+jb3wsjEZ8vPn3nUam4caLelYG8GylXUnQJkqJ8bEzCaOOnKeWVVPy+QZvN1upVgC
+Zip8qlWqJYkKcKMNawLlkw51yeEXssbvwCHVYSazR0hLv3kvckU96DcCgYEAx/0r
+DBsb0DjmG/QQRTfkZ/TpifgWCEzuw5TQrONPwExRKxMmfYuT6mVuVPHxA6GH4y2R
+jdDl/CQT/iiS/dOxBDg8Z4W89BGm7YTf8OFUtknDAXlTYAmp6wAgbHrps5wYRPQY
+eYfvNRByWg1Vf4RTAWLHV7yNnmajFc/CmCpOpM0CgYEAteItChIIMkZNxQacJet7
+gooJ6ddezeBNL3rggJqE+c1nk6IoGCPLQCo6N00zTyNC82Aj3uYVtimiKRTljHj4
+ekltkvBCpSI6IaXm/24lN05mryauxA5lMYDjuPj9J/sCuGh+Mkjod3y1HuiJPBAE
+4ZqCW1P8TOoZHQODs3KDvi8CgYEAp3TZgr2AQI4kP+vRY+Q1IMb2dkyBRloE40fM
+o7eCo+ZDYOocJKunI3HjPMWJphyviA4z/qeYq9QZ2Ytn6ZpEcuAT1Csr2MKfH+tP
+CIflQu3ol2db5OKyoM/uVmdlILvnpnUtuTWVmr02wyi/GpNWXaa0fEap3qTakfzx
+WDC0wVECgYAV6plB4s007I6d6SgBYAQrUMSMlzNO8oTUAI8Zc/PEpQef0WMO7/11
+MQDD25LTegbxW7+DT8Qwhh0U7MWO4dr0hl1QdiMpTYw0GWV9HpPx8ta37W3Jx/0i
+SrdQUEZ3rP/HXcQsW7nNBgplI+nRYL3eoVXAT9PgF0ug+EdFzHUcUQ==
+-----END RSA PRIVATE KEY-----

dev/ssh-host/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCn2r5IZSJp02FBAYSZBQRdOBKBK2VOErdrBCZm5Ig3hDKQuxq38+W5CJ2JUJU+LQm//uenm58scGlEtk5+w5SjObjzK8Qx6JeRhAiZ8xpyydSoUIvd0ARD9OKwdiQFqVlLPlOyrdIpQ2vRESdwzhe0f7EYUwgKzBw5k0foxQsGxTiztY/ugWJ8Jso5WOxXwzEw4cSnGhdrehqLphlZanr54wj5oTcrj/vJHlpbxkYzFMc2Zgj81GdIV4yP3H1yX4ySK8VkDPOCczHacdRnHw4u8Vgf6wS6Zy3iMpvuGu7BJkwNoTXvmVV5BXUm6GAMSQTAPcw5T8M+eXjSAnriGDAL

dev/ssh-host/known_hosts

@@ -0,0 +1 @@
+ssh-host ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCunjxQLDBHohVN13u/fMQNlHeM+QQK6N6LRNo6XO/Y08CKIh6s7YFmZdPWqslOilLdGQM8z8ShZd/WrCjGEbanYQI2c5TgduLd6CIjL3t07otLtf8KqXZCDBBtOvvlTUWHJQk1hN3STdfw+VNjdOMhLeWTtCAyO2zgJQTfkqrrjVul+m3ykAOQw4ULLpTefrZ2/qpcbJaDl8qoortpMuhHExxvMgJJhyx0cOLHA7UdEgEr+2BfMTj6BznB+udREYTFDFTgkDexHzdpptphtO2HCqyY06z8lacdOPw5mXe0Ilfr/EFDhtkk4i8MsdRLhaqtkzvw914t/4yZcFBDd1DNQzMcBK4W8WZUNeArsB14/UMhACFj2QUIGyxa8yoawQ5G8EaEf0Djg1MP+gnTFb2fu9vXBEO0Bu/TYOIfs9W9iKN5aw7NvupulCcO1eTQED3k5QIKeautKL42hnPCnL8SQQsS2JPRRzXtarjLIghog1chirQNFkfNiNjsa2ltOp0=

dev/ssh-host/ssh_host_rsa_key

@@ -0,0 +1,39 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEArp48UCwwR6IVTdd7v3zEDZR3jPkECujei0TaOlzv2NPAiiIerO2B
+ZmXT1qrJTopS3RkDPM/EoWXf1qwoxhG2p2ECNnOU4Hbi3egiIy97dO6LS7X/Cql2QgwQbT
+r75U1FhyUJNYTd0k3X8PlTY3TjIS3lk7QgMjts4CUE35Kq641bpfpt8pADkMOFCy6U3n62
+dv6qXGyWg5fKqKK7aTLoRxMcbzICSYcsdHDixwO1HRIBK/tgXzE4+gc5wfrnURGExQxU4J
+A3sR83aabaYbTthwqsmNOs/JWnHTj8OZl3tCJX6/xBQ4bZJOIvDLHUS4WqrZM78PdeLf+M
+mXBQQ3dQzUMzHASuFvFmVDXgK7AdeP1DIQAhY9kFCBssWvMqGsEORvBGhH9A44NTD/oJ0x
+W9n7vb1wRDtAbv02DiH7PVvYijeWsOzb7qbpQnDtXk0BA95OUCCnmrrSi+NoZzwpy/EkEL
+EtiT0Uc17Wq4yyIIaINXIYq0DRZHzYjY7GtpbTqdAAAFoKgeDg2oHg4NAAAAB3NzaC1yc2
+EAAAGBAK6ePFAsMEeiFU3Xe798xA2Ud4z5BAro3otE2jpc79jTwIoiHqztgWZl09aqyU6K
+Ut0ZAzzPxKFl39asKMYRtqdhAjZzlOB24t3oIiMve3Tui0u1/wqpdkIMEG06++VNRYclCT
+WE3dJN1/D5U2N04yEt5ZO0IDI7bOAlBN+SquuNW6X6bfKQA5DDhQsulN5+tnb+qlxsloOX
+yqiiu2ky6EcTHG8yAkmHLHRw4scDtR0SASv7YF8xOPoHOcH651ERhMUMVOCQN7EfN2mm2m
+G07YcKrJjTrPyVpx04/DmZd7QiV+v8QUOG2STiLwyx1EuFqq2TO/D3Xi3/jJlwUEN3UM1D
+MxwErhbxZlQ14CuwHXj9QyEAIWPZBQgbLFrzKhrBDkbwRoR/QOODUw/6CdMVvZ+729cEQ7
+QG79Ng4h+z1b2Io3lrDs2+6m6UJw7V5NAQPeTlAgp5q60ovjaGc8KcvxJBCxLYk9FHNe1q
+uMsiCGiDVyGKtA0WR82I2OxraW06nQAAAAMBAAEAAAGACdCno7sEILal22kCEe6dp4TBno
+kttr5Fqg7d9FteePHYF/uYfVBhTmPpXx7c7177juV1xuCH1SmghhTJuu5qdaiQgwaGpwJP
+uLjwWEl2N0mkR0Zs1kjVtpsufjFLUOWBw7mrdZhpDoXlHiypiQTcMnR9u8prZ99qvIOgLT
+/1fwWEUgVMUk7RgHzY+Nqur/3v3Cru4QCSikWJNObmwWBE6Z/ToJVvRvpD36yrtpOJBeAJ
+9FKuJVOjN/yZfMORZn9lPu0HTbkyWw2hYN7A6I2oXsHFuCz1VDnFdjv4xHCBFwPu/mzXvG
+RkhYQHr5YzXohn+SJUBCvTqGgl/PXvTrD8aw5SQmBINEZXtBlL9a4UKiPmtS5QcOICa4ps
+eODzAiY01jTntb0uJh9lJTL5QK9Aq2W0VJHRL8cQFiyMYAw0vL1l2Lk/XTomcxAyW98ABd
+pbFZxNfbkHd6YbQQ7UEZLUnvcs7+OXBe0lLtrM38TNKlcqVepHPel0RJKu9zU8iEa5AAAA
+wQDn6RUSt5jY7cWd8hbmomtwZYdcSqo9ySkUVINGi13qNBmIwtLK4rMV5zng65tNZFeOjh
+646dPVBzyxfR9I2hior0OMcswzNfLXZx90AJAEzrpoFmxQS59yP2GOgBY8SSDo6V5Bgm61
+qMyAMxWk9+qoqGpKCqvbIY5Oq0tEhv/VF3TTipvcqWw7jg+3Yopw/4sG0XtFiQa3hqC3LH
+zxeoW3U4su94M+NExwoXOGsUoDHyunJUWiY83RrK/G1Pf7zzAAAADBAO48ES7Dt8rUxiqQ
+fUsWzdL2lX1ASgYqkpmfSkivV93L9kL+eRTHGzov1EStezI8Gx8NBpzpE7f8z6wftlvlya
+dCnq2hmQJLVDTIZlbgT6AQ9VBs7FZP0ZmRSP49uV8JY/dEOqi+o6xaDHflsPF/tx5pC6KV
+WPxgoqrkmnwt7vgwmBd0vY75rgdxsEW/S1B8AvU43eG1Bi4HaqRHDeS2ffOCOI4dOkzU1z
+MyFIpIn0Sx7qXlRLdppu9JSz9ssCZR1QAAAMEAu6O4TlW+3IuQOTkpNPKHtS40VW9Rk4De
+995+ktFQFRbYeHptJWAs+abbTZfV6XBNsMx9z6C1x2J9v7/O7/lqGMyPCBsUn//FrtjA+l
+GUDCutEYiMewKsZpMzEY5XGbGV4srSHazrlYgVDi1MjWSo5zYG3EloJe/wrew26y76mREM
+yA9u+SLVJ3NBtFYtTkf7WecH2xtT2H6uedT+QeisVFKFpzdwIvyNUswgqS7S6vu8QL3nRM
+7twwb3BPpu4eGpAAAAI2Jlbm9pdEBVYnVudHUtMjIwNC1qYW1teS1hbWQ2NC1iYXNlAQID
+BAUGBw==
+-----END OPENSSH PRIVATE KEY-----

uploader/CONTRIBUTING.md

@@ -0,0 +1,59 @@
+# Contributing
+
+In order to test this component, you need:
+- Docker
+- a test Zimfarm instance with a username + password (below we use the one from `dev` folder Docker compose stack)
+- a test SSH host (below we use the one from `dev` folder Docker compose stack)
+- a test S3 bucket on Wasabi (you will have to set compliance to fully test the uploader)
+- credentials to access this bucket (keyId and secretAccessKey suggested below)
+
+Rebuild the Docker image:
+
+```
+docker build -t local-zf-uploader .
+```
+
+## S3 tests
+
+Export the secret `S3_URL` as environment variable. Note that the S3 URL starts with `s3`.
+
+
+On Bash/Zsh shells (replace `<your_s3_host_name>`, `<your_key_id>`, `<your_secret_access_key>` and `<your_bucket>` with proper values):
+
+```
+ export S3_URL="s3://<your_s3_host_name>/?keyId=<your_key_id>&secretAccessKey=<your_secret_access_key>&bucketName=<your_bucket>"
+```
+
+Run a test without compliance activated on the bucket:
+
+```
+docker run -it --rm -v $PWD:/data local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri $S3_URL
+```
+
+Run same test twice, it should work.
+
+Delete the file from the bucket and activate compliance (with a 1 day setting for instance).
+
+Run a test which will set the object compliance:
+
+```
+docker run -it --rm -v $PWD:/data local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri $S3_URL --delete-after 30
+```
+
+It should succeed. If you run the same test a second time, it will fail due to compliance (HTTP 400 error).
+
+## SCP test
+
+Following test upload should succeed.
+
+```
+docker run -it --rm -v $PWD:/data -v $PWD/../dev/ssh-host/id_rsa:/etc/ssh/keys/id_rsa -v $PWD/../dev/ssh-host/known_hosts:/etc/ssh/known_hosts --network zimfarm_default local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri scp://root@ssh-host:22/CONTRIBUTING.md --move
+```
+
+## SFTP test
+
+Following test upload should succeed.
+
+```
+docker run -it --rm -v $PWD:/data -v $PWD/../dev/ssh-host/id_rsa:/etc/ssh/keys/id_rsa -v $PWD/../dev/ssh-host/known_hosts:/etc/ssh/known_hosts --network zimfarm_default local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri sftp://root@ssh-host:22/CONTRIBUTING.md
+```

uploader/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-buster
+FROM python:3.12-slim-bookworm
 LABEL zimfarm=true
 LABEL org.opencontainers.image.source https://github.com/openzim/zimfarm
 

uploader/requirements.txt

@@ -1 +1 @@
-kiwixstorage>=0.8.3,<0.9
+kiwixstorage==0.8.3

uploader/setup.py

@@ -45,12 +45,8 @@ def read(*names, **kwargs):
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.12",
         "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
     ],
-    python_requires=">=3.6",
+    python_requires=">=3.12,<3.13",
 )

workers/app/common/docker.py

@@ -614,24 +614,26 @@ def start_uploader(
     if task["upload"][kind]["expiration"]:
         command += ["--delete-after", str(task["upload"][kind]["expiration"])]
 
-    return run_container(
-        docker_client,
-        image=docker_image,
-        command=command,
-        detach=True,
-        environment={"RSA_KEY": str(PRIVATE_KEY)},
-        labels={
+    kwargs = {
+        "image": docker_image,
+        "command": command,
+        "detach": True,
+        "environment": {"RSA_KEY": str(PRIVATE_KEY)},
+        "labels": {
             "zimfarm": "",
             "task_id": task["_id"],
             "tid": short_id(task["_id"]),
             "schedule_name": task["schedule_name"],
             "filename": filename,
         },
-        mem_swappiness=0,
-        mounts=mounts,
-        name=container_name,
-        remove=False,
-    )
+        "mem_swappiness": 0,
+        "mounts": mounts,
+        "name": container_name,
+        "remove": False,
+    }
+    if "DOCKER_NETWORK" in os.environ:
+        kwargs["network"] = os.getenv("DOCKER_NETWORK")
+    return run_container(docker_client, **kwargs)
 
 
 def get_container_logs(docker_client, container_name, tail="all"):