From d06532e542b9dfe3d18dbb0448ed0a60d88df00d Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 22 Feb 2024 17:03:09 +0100 Subject: [PATCH 1/5] Upgrade uploader dependencies => Python 3.12 and Debian slim-bookworm --- uploader/Dockerfile | 2 +- uploader/setup.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/uploader/Dockerfile b/uploader/Dockerfile index 71ac2333c..c028914c2 100644 --- a/uploader/Dockerfile +++ b/uploader/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-buster +FROM python:3.12-slim-bookworm LABEL zimfarm=true LABEL org.opencontainers.image.source https://github.com/openzim/zimfarm diff --git a/uploader/setup.py b/uploader/setup.py index 87c1e562d..4c22e4d5e 100644 --- a/uploader/setup.py +++ b/uploader/setup.py @@ -45,12 +45,8 @@ def read(*names, **kwargs): "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ], - python_requires=">=3.6", + python_requires=">=3.12,<3.13", ) From 830ef1ef5e788e1e5bab7ba80b8bed7f3afd689d Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 22 Feb 2024 17:07:00 +0100 Subject: [PATCH 2/5] Pin uploader dependency according to new Python convention --- uploader/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uploader/requirements.txt b/uploader/requirements.txt index 46100481b..7b84db472 100644 --- a/uploader/requirements.txt +++ b/uploader/requirements.txt @@ -1 +1 @@ -kiwixstorage>=0.8.3,<0.9 +kiwixstorage==0.8.3 From e5cdb5cfd28a95249351f88e112f470d42407fff Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 26 Feb 2024 11:24:57 +0100 Subject: [PATCH 3/5] Add a test SSH host for uploading files --- dev/README.md | 4 ++++ dev/docker-compose.yml | 10 ++++++++ dev/ssh-host/Dockerfile | 44 +++++++++++++++++++++++++++++++++++ dev/ssh-host/id_rsa | 27 +++++++++++++++++++++ dev/ssh-host/id_rsa.pub | 1 + dev/ssh-host/known_hosts | 1 + dev/ssh-host/ssh_host_rsa_key | 39 +++++++++++++++++++++++++++++++ 7 files changed, 126 insertions(+) create mode 100644 dev/ssh-host/Dockerfile create mode 100644 dev/ssh-host/id_rsa create mode 100644 dev/ssh-host/id_rsa.pub create mode 100644 dev/ssh-host/known_hosts create mode 100644 dev/ssh-host/ssh_host_rsa_key diff --git a/dev/README.md b/dev/README.md index 2f580fd6f..7aee77391 100644 --- a/dev/README.md +++ b/dev/README.md @@ -29,6 +29,10 @@ This container is a PostgreSQL DB. DB data is kept in a volume, persistent acros This container hosts the frontend UI for end-users. +### ssh-host + +This container hosts a minimal SSH server, usefull for uploading Zimfarm artifacts locally during tests. + ## Instructions First start the Docker-Compose stack: diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index b748c6016..4f4e5e2f3 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -28,6 +28,9 @@ services: JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@postgresdb:5432/zimfarm ALEMBIC_UPGRADE_HEAD_ON_START: "1" + ARTIFACTS_UPLOAD_URI: scp://root@ssh-host:22/artifacts/ + LOGS_UPLOAD_URI: scp://root@ssh-host:22/logs/ + ZIM_UPLOAD_URI: scp://root@ssh-host:22/zims/ depends_on: - postgresdb frontend-ui: @@ -78,6 +81,13 @@ services: POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@postgresdb:5432/zimtest depends_on: - postgresdb + ssh-host: + build: + context: ssh-host + container_name: zf_ssh_host + ports: + - 127.0.0.1:8022:22 + volumes: pg_data_zimfarm: diff --git a/dev/ssh-host/Dockerfile b/dev/ssh-host/Dockerfile new file mode 100644 index 000000000..d0d32ca13 --- /dev/null +++ b/dev/ssh-host/Dockerfile @@ -0,0 +1,44 @@ +FROM alpine:3 + +# Install SSH server +RUN apk update && apk add --no-cache openssh-server + +# Create SSH directory and set permissions +RUN mkdir /var/run/sshd +RUN chmod 0755 /var/run/sshd + +# Copy SSH host keys to container +COPY ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key + +# Copy test client SSH public key to the container +COPY id_rsa.pub /root/.ssh/authorized_keys + +RUN mkdir -p \ + /root/artifacts \ + /root/logs \ + /root/zims/freecodecamp \ + /root/zims/gutenberg \ + /root/zims/ifixit \ + /root/zims/mooc \ + /root/zims/other \ + /root/zims/phet \ + /root/zims/stack_exchange \ + /root/zims/ted \ + /root/zims/videos \ + /root/zims/vikidia \ + /root/zims/wikibooks \ + /root/zims/wikihow \ + /root/zims/wikinews \ + /root/zims/wikipedia \ + /root/zims/wikiquote \ + /root/zims/wikisource \ + /root/zims/wikiversity \ + /root/zims/wikivoyage \ + /root/zims/wiktionary \ + /root/zims/zimit + +# Expose SSH port +EXPOSE 22 + +# Start SSH server +CMD ["/usr/sbin/sshd", "-D"] \ No newline at end of file diff --git a/dev/ssh-host/id_rsa b/dev/ssh-host/id_rsa new file mode 100644 index 000000000..c06758d6c --- /dev/null +++ b/dev/ssh-host/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAp9q+SGUiadNhQQGEmQUEXTgSgStlThK3awQmZuSIN4QykLsa +t/PluQidiVCVPi0Jv/7np5ufLHBpRLZOfsOUozm48yvEMeiXkYQImfMacsnUqFCL +3dAEQ/TisHYkBalZSz5Tsq3SKUNr0REncM4XtH+xGFMICswcOZNH6MULBsU4s7WP +7oFifCbKOVjsV8MxMOHEpxoXa3oai6YZWWp6+eMI+aE3K4/7yR5aW8ZGMxTHNmYI +/NRnSFeMj9x9cl+MkivFZAzzgnMx2nHUZx8OLvFYH+sEumct4jKb7hruwSZMDaE1 +75lVeQV1JuhgDEkEwD3MOU/DPnl40gJ64hgwCwIDAQABAoIBAAQzoQy931knzejr +KU8NRZkxxKDQHcaRCF9Y6L8tnE1LdVqTHG2bYNY7ZXog9sNqLDLOpizXY+ogXDo4 +mAlrSua1FLCdmVkoedbOJ8r2v2Q9MiIOdhnU0a6DzELfijJMFWyhQEYaPMcEpYfD +tqmHtycxBRVXSr2+czxrt+KXQBnovoqkM3M/ritAyj6HhQ8ZgqecmKA0UNWHXMX9 +fYJqeHtoek7DOM30C2qUOfiTE8Cd92BFyFffajL13tZPS0hD1yN3rE11khCbMfMo +5n4lKykBw4G6EX4+mzTLPNxK29sead9IYfa0RteJIVXZG4GX7wnok8vVLm2f6k0N +iMsNPMUCgYEA1t2X/7y/x/RMOpZ3dDvK2Zm3bnICah7nXK7afs/WFUiwkzFg92S1 +jb3wsjEZ8vPn3nUam4caLelYG8GylXUnQJkqJ8bEzCaOOnKeWVVPy+QZvN1upVgC +Zip8qlWqJYkKcKMNawLlkw51yeEXssbvwCHVYSazR0hLv3kvckU96DcCgYEAx/0r +DBsb0DjmG/QQRTfkZ/TpifgWCEzuw5TQrONPwExRKxMmfYuT6mVuVPHxA6GH4y2R +jdDl/CQT/iiS/dOxBDg8Z4W89BGm7YTf8OFUtknDAXlTYAmp6wAgbHrps5wYRPQY +eYfvNRByWg1Vf4RTAWLHV7yNnmajFc/CmCpOpM0CgYEAteItChIIMkZNxQacJet7 +gooJ6ddezeBNL3rggJqE+c1nk6IoGCPLQCo6N00zTyNC82Aj3uYVtimiKRTljHj4 +ekltkvBCpSI6IaXm/24lN05mryauxA5lMYDjuPj9J/sCuGh+Mkjod3y1HuiJPBAE +4ZqCW1P8TOoZHQODs3KDvi8CgYEAp3TZgr2AQI4kP+vRY+Q1IMb2dkyBRloE40fM +o7eCo+ZDYOocJKunI3HjPMWJphyviA4z/qeYq9QZ2Ytn6ZpEcuAT1Csr2MKfH+tP +CIflQu3ol2db5OKyoM/uVmdlILvnpnUtuTWVmr02wyi/GpNWXaa0fEap3qTakfzx +WDC0wVECgYAV6plB4s007I6d6SgBYAQrUMSMlzNO8oTUAI8Zc/PEpQef0WMO7/11 +MQDD25LTegbxW7+DT8Qwhh0U7MWO4dr0hl1QdiMpTYw0GWV9HpPx8ta37W3Jx/0i +SrdQUEZ3rP/HXcQsW7nNBgplI+nRYL3eoVXAT9PgF0ug+EdFzHUcUQ== +-----END RSA PRIVATE KEY----- diff --git a/dev/ssh-host/id_rsa.pub b/dev/ssh-host/id_rsa.pub new file mode 100644 index 000000000..2b75c178d --- /dev/null +++ b/dev/ssh-host/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCn2r5IZSJp02FBAYSZBQRdOBKBK2VOErdrBCZm5Ig3hDKQuxq38+W5CJ2JUJU+LQm//uenm58scGlEtk5+w5SjObjzK8Qx6JeRhAiZ8xpyydSoUIvd0ARD9OKwdiQFqVlLPlOyrdIpQ2vRESdwzhe0f7EYUwgKzBw5k0foxQsGxTiztY/ugWJ8Jso5WOxXwzEw4cSnGhdrehqLphlZanr54wj5oTcrj/vJHlpbxkYzFMc2Zgj81GdIV4yP3H1yX4ySK8VkDPOCczHacdRnHw4u8Vgf6wS6Zy3iMpvuGu7BJkwNoTXvmVV5BXUm6GAMSQTAPcw5T8M+eXjSAnriGDAL diff --git a/dev/ssh-host/known_hosts b/dev/ssh-host/known_hosts new file mode 100644 index 000000000..bdd5f2f79 --- /dev/null +++ b/dev/ssh-host/known_hosts @@ -0,0 +1 @@ +ssh-host ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCunjxQLDBHohVN13u/fMQNlHeM+QQK6N6LRNo6XO/Y08CKIh6s7YFmZdPWqslOilLdGQM8z8ShZd/WrCjGEbanYQI2c5TgduLd6CIjL3t07otLtf8KqXZCDBBtOvvlTUWHJQk1hN3STdfw+VNjdOMhLeWTtCAyO2zgJQTfkqrrjVul+m3ykAOQw4ULLpTefrZ2/qpcbJaDl8qoortpMuhHExxvMgJJhyx0cOLHA7UdEgEr+2BfMTj6BznB+udREYTFDFTgkDexHzdpptphtO2HCqyY06z8lacdOPw5mXe0Ilfr/EFDhtkk4i8MsdRLhaqtkzvw914t/4yZcFBDd1DNQzMcBK4W8WZUNeArsB14/UMhACFj2QUIGyxa8yoawQ5G8EaEf0Djg1MP+gnTFb2fu9vXBEO0Bu/TYOIfs9W9iKN5aw7NvupulCcO1eTQED3k5QIKeautKL42hnPCnL8SQQsS2JPRRzXtarjLIghog1chirQNFkfNiNjsa2ltOp0= diff --git a/dev/ssh-host/ssh_host_rsa_key b/dev/ssh-host/ssh_host_rsa_key new file mode 100644 index 000000000..5516b92e5 --- /dev/null +++ b/dev/ssh-host/ssh_host_rsa_key @@ -0,0 +1,39 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEArp48UCwwR6IVTdd7v3zEDZR3jPkECujei0TaOlzv2NPAiiIerO2B +ZmXT1qrJTopS3RkDPM/EoWXf1qwoxhG2p2ECNnOU4Hbi3egiIy97dO6LS7X/Cql2QgwQbT +r75U1FhyUJNYTd0k3X8PlTY3TjIS3lk7QgMjts4CUE35Kq641bpfpt8pADkMOFCy6U3n62 +dv6qXGyWg5fKqKK7aTLoRxMcbzICSYcsdHDixwO1HRIBK/tgXzE4+gc5wfrnURGExQxU4J +A3sR83aabaYbTthwqsmNOs/JWnHTj8OZl3tCJX6/xBQ4bZJOIvDLHUS4WqrZM78PdeLf+M +mXBQQ3dQzUMzHASuFvFmVDXgK7AdeP1DIQAhY9kFCBssWvMqGsEORvBGhH9A44NTD/oJ0x +W9n7vb1wRDtAbv02DiH7PVvYijeWsOzb7qbpQnDtXk0BA95OUCCnmrrSi+NoZzwpy/EkEL +EtiT0Uc17Wq4yyIIaINXIYq0DRZHzYjY7GtpbTqdAAAFoKgeDg2oHg4NAAAAB3NzaC1yc2 +EAAAGBAK6ePFAsMEeiFU3Xe798xA2Ud4z5BAro3otE2jpc79jTwIoiHqztgWZl09aqyU6K +Ut0ZAzzPxKFl39asKMYRtqdhAjZzlOB24t3oIiMve3Tui0u1/wqpdkIMEG06++VNRYclCT +WE3dJN1/D5U2N04yEt5ZO0IDI7bOAlBN+SquuNW6X6bfKQA5DDhQsulN5+tnb+qlxsloOX +yqiiu2ky6EcTHG8yAkmHLHRw4scDtR0SASv7YF8xOPoHOcH651ERhMUMVOCQN7EfN2mm2m +G07YcKrJjTrPyVpx04/DmZd7QiV+v8QUOG2STiLwyx1EuFqq2TO/D3Xi3/jJlwUEN3UM1D +MxwErhbxZlQ14CuwHXj9QyEAIWPZBQgbLFrzKhrBDkbwRoR/QOODUw/6CdMVvZ+729cEQ7 +QG79Ng4h+z1b2Io3lrDs2+6m6UJw7V5NAQPeTlAgp5q60ovjaGc8KcvxJBCxLYk9FHNe1q +uMsiCGiDVyGKtA0WR82I2OxraW06nQAAAAMBAAEAAAGACdCno7sEILal22kCEe6dp4TBno +kttr5Fqg7d9FteePHYF/uYfVBhTmPpXx7c7177juV1xuCH1SmghhTJuu5qdaiQgwaGpwJP +uLjwWEl2N0mkR0Zs1kjVtpsufjFLUOWBw7mrdZhpDoXlHiypiQTcMnR9u8prZ99qvIOgLT +/1fwWEUgVMUk7RgHzY+Nqur/3v3Cru4QCSikWJNObmwWBE6Z/ToJVvRvpD36yrtpOJBeAJ +9FKuJVOjN/yZfMORZn9lPu0HTbkyWw2hYN7A6I2oXsHFuCz1VDnFdjv4xHCBFwPu/mzXvG +RkhYQHr5YzXohn+SJUBCvTqGgl/PXvTrD8aw5SQmBINEZXtBlL9a4UKiPmtS5QcOICa4ps +eODzAiY01jTntb0uJh9lJTL5QK9Aq2W0VJHRL8cQFiyMYAw0vL1l2Lk/XTomcxAyW98ABd +pbFZxNfbkHd6YbQQ7UEZLUnvcs7+OXBe0lLtrM38TNKlcqVepHPel0RJKu9zU8iEa5AAAA +wQDn6RUSt5jY7cWd8hbmomtwZYdcSqo9ySkUVINGi13qNBmIwtLK4rMV5zng65tNZFeOjh +646dPVBzyxfR9I2hior0OMcswzNfLXZx90AJAEzrpoFmxQS59yP2GOgBY8SSDo6V5Bgm61 +qMyAMxWk9+qoqGpKCqvbIY5Oq0tEhv/VF3TTipvcqWw7jg+3Yopw/4sG0XtFiQa3hqC3LH +zxeoW3U4su94M+NExwoXOGsUoDHyunJUWiY83RrK/G1Pf7zzAAAADBAO48ES7Dt8rUxiqQ +fUsWzdL2lX1ASgYqkpmfSkivV93L9kL+eRTHGzov1EStezI8Gx8NBpzpE7f8z6wftlvlya +dCnq2hmQJLVDTIZlbgT6AQ9VBs7FZP0ZmRSP49uV8JY/dEOqi+o6xaDHflsPF/tx5pC6KV +WPxgoqrkmnwt7vgwmBd0vY75rgdxsEW/S1B8AvU43eG1Bi4HaqRHDeS2ffOCOI4dOkzU1z +MyFIpIn0Sx7qXlRLdppu9JSz9ssCZR1QAAAMEAu6O4TlW+3IuQOTkpNPKHtS40VW9Rk4De +995+ktFQFRbYeHptJWAs+abbTZfV6XBNsMx9z6C1x2J9v7/O7/lqGMyPCBsUn//FrtjA+l +GUDCutEYiMewKsZpMzEY5XGbGV4srSHazrlYgVDi1MjWSo5zYG3EloJe/wrew26y76mREM +yA9u+SLVJ3NBtFYtTkf7WecH2xtT2H6uedT+QeisVFKFpzdwIvyNUswgqS7S6vu8QL3nRM +7twwb3BPpu4eGpAAAAI2Jlbm9pdEBVYnVudHUtMjIwNC1qYW1teS1hbWQ2NC1iYXNlAQID +BAUGBw== +-----END OPENSSH PRIVATE KEY----- From 791d331e08f6910227afa2396c964df992a02320 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 26 Feb 2024 11:26:37 +0100 Subject: [PATCH 4/5] Add CONTRIBUTING guide for uploader --- uploader/CONTRIBUTING.md | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 uploader/CONTRIBUTING.md diff --git a/uploader/CONTRIBUTING.md b/uploader/CONTRIBUTING.md new file mode 100644 index 000000000..f5cb3e485 --- /dev/null +++ b/uploader/CONTRIBUTING.md @@ -0,0 +1,59 @@ +# Contributing + +In order to test this component, you need: +- Docker +- a test Zimfarm instance with a username + password (below we use the one from `dev` folder Docker compose stack) +- a test SSH host (below we use the one from `dev` folder Docker compose stack) +- a test S3 bucket on Wasabi (you will have to set compliance to fully test the uploader) +- credentials to access this bucket (keyId and secretAccessKey suggested below) + +Rebuild the Docker image: + +``` +docker build -t local-zf-uploader . +``` + +## S3 tests + +Export the secret `S3_URL` as environment variable. Note that the S3 URL starts with `s3`. + + +On Bash/Zsh shells (replace ``, ``, `` and `` with proper values): + +``` + export S3_URL="s3:///?keyId=&secretAccessKey=&bucketName=" +``` + +Run a test without compliance activated on the bucket: + +``` +docker run -it --rm -v $PWD:/data local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri $S3_URL +``` + +Run same test twice, it should work. + +Delete the file from the bucket and activate compliance (with a 1 day setting for instance). + +Run a test which will set the object compliance: + +``` +docker run -it --rm -v $PWD:/data local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri $S3_URL --delete-after 30 +``` + +It should succeed. If you run the same test a second time, it will fail due to compliance (HTTP 400 error). + +## SCP test + +Following test upload should succeed. + +``` +docker run -it --rm -v $PWD:/data -v $PWD/../dev/ssh-host/id_rsa:/etc/ssh/keys/id_rsa -v $PWD/../dev/ssh-host/known_hosts:/etc/ssh/known_hosts --network zimfarm_default local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri scp://root@ssh-host:22/CONTRIBUTING.md --move +``` + +## SFTP test + +Following test upload should succeed. + +``` +docker run -it --rm -v $PWD:/data -v $PWD/../dev/ssh-host/id_rsa:/etc/ssh/keys/id_rsa -v $PWD/../dev/ssh-host/known_hosts:/etc/ssh/known_hosts --network zimfarm_default local-zf-uploader uploader --file /data/CONTRIBUTING.md --upload-uri sftp://root@ssh-host:22/CONTRIBUTING.md +``` \ No newline at end of file From e89955a365bc6fdde10a16b14de8956c9c162c4a Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 26 Feb 2024 15:03:18 +0100 Subject: [PATCH 5/5] Allow to pass docker network to child task manager containers In rare occasions (specific docker setup, local development stack), we might want to run task manager child containers (uploaders) in a specific Docker network. This adds an environment variable to set in the task manager which will be further used for children containers (uploaders) --- workers/app/common/docker.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/workers/app/common/docker.py b/workers/app/common/docker.py index 7b64867e2..e0cf9cfc4 100644 --- a/workers/app/common/docker.py +++ b/workers/app/common/docker.py @@ -614,24 +614,26 @@ def start_uploader( if task["upload"][kind]["expiration"]: command += ["--delete-after", str(task["upload"][kind]["expiration"])] - return run_container( - docker_client, - image=docker_image, - command=command, - detach=True, - environment={"RSA_KEY": str(PRIVATE_KEY)}, - labels={ + kwargs = { + "image": docker_image, + "command": command, + "detach": True, + "environment": {"RSA_KEY": str(PRIVATE_KEY)}, + "labels": { "zimfarm": "", "task_id": task["_id"], "tid": short_id(task["_id"]), "schedule_name": task["schedule_name"], "filename": filename, }, - mem_swappiness=0, - mounts=mounts, - name=container_name, - remove=False, - ) + "mem_swappiness": 0, + "mounts": mounts, + "name": container_name, + "remove": False, + } + if "DOCKER_NETWORK" in os.environ: + kwargs["network"] = os.getenv("DOCKER_NETWORK") + return run_container(docker_client, **kwargs) def get_container_logs(docker_client, container_name, tail="all"):