From d65d866292370c41025a7f7dcdda304746ce0c59 Mon Sep 17 00:00:00 2001 From: South Drifted Date: Thu, 5 Oct 2023 15:41:38 +0000 Subject: [PATCH 01/10] feat: add Docker configuration files --- Dockerfile | 19 +++++++++++++++++++ docker-compose.yml | 26 ++++++++++++++++++++++++++ package-lock.json | 8 +++----- package.json | 2 +- 4 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..af72359 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM node:18-slim + +USER root + +RUN npm rm yarn -g + +RUN mkdir /home/node/app +WORKDIR /home/node/app + +COPY package.json package-lock.json file-content-injector.js /home/node/app/ +RUN npm i + +COPY . /home/node/app + +RUN npm prune --production || true \ + npm cache clean -f + +EXPOSE 3000 +CMD ["npm", "start"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..879baf4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3' + +networks: + polyfiller: + +services: + api-service: + image: polyfiller/api-service + environment: + - NODE_ENV=production + ports: + - 3000:3000 + networks: + - polyfiller + restart: always + caddy: + depends_on: + - api-service + image: caddy + ports: + - 80:80 + - 443:443 + networks: + - polyfiller + restart: always + command: caddy reverse-proxy --from polyfiller.app --to api-service:3000 diff --git a/package-lock.json b/package-lock.json index a213f86..4c16ed5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "@wessberg/di": "2.1.0", "@wessberg/pointer-events": "1.0.9", "@wessberg/stringutil": "1.0.19", + "@wessberg/ts-config": "3.1.0", "blob-polyfill": "5.0.20210201", "browserslist": "4.21.5", "browserslist-generator": "2.0.3", @@ -84,7 +85,6 @@ "@typescript-eslint/parser": "5.48.2", "@wessberg/di-compiler": "3.2.0", "@wessberg/prettier-config": "1.0.0", - "@wessberg/ts-config": "3.1.0", "ava": "5.1.1", "eslint": "8.32.0", "eslint-config-prettier": "8.6.0", @@ -3632,8 +3632,7 @@ "node_modules/@wessberg/ts-config": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@wessberg/ts-config/-/ts-config-3.1.0.tgz", - "integrity": "sha512-/TqKHx3zgdDQKoqAemh4tBYQE873aN37BuaTeE5AFC3ZG/9oSVgkbuu32WErFsSHAnpdblpBuYJwhugUm0r7Xg==", - "dev": true + "integrity": "sha512-/TqKHx3zgdDQKoqAemh4tBYQE873aN37BuaTeE5AFC3ZG/9oSVgkbuu32WErFsSHAnpdblpBuYJwhugUm0r7Xg==" }, "node_modules/accepts": { "version": "1.3.7", @@ -18350,8 +18349,7 @@ "@wessberg/ts-config": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@wessberg/ts-config/-/ts-config-3.1.0.tgz", - "integrity": "sha512-/TqKHx3zgdDQKoqAemh4tBYQE873aN37BuaTeE5AFC3ZG/9oSVgkbuu32WErFsSHAnpdblpBuYJwhugUm0r7Xg==", - "dev": true + "integrity": "sha512-/TqKHx3zgdDQKoqAemh4tBYQE873aN37BuaTeE5AFC3ZG/9oSVgkbuu32WErFsSHAnpdblpBuYJwhugUm0r7Xg==" }, "accepts": { "version": "1.3.7", diff --git a/package.json b/package.json index bae7bfb..b7094b9 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,6 @@ "@typescript-eslint/parser": "5.48.2", "@wessberg/di-compiler": "3.2.0", "@rollup/plugin-json": "6.0.0", - "@wessberg/ts-config": "3.1.0", "@wessberg/prettier-config": "1.0.0", "ava": "5.1.1", "eslint": "8.32.0", @@ -93,6 +92,7 @@ "@wessberg/di": "2.1.0", "@wessberg/pointer-events": "1.0.9", "@wessberg/stringutil": "1.0.19", + "@wessberg/ts-config": "3.1.0", "lit": "2.6.1", "blob-polyfill": "5.0.20210201", "browserslist": "4.21.5", From 8fc5dd85c2d2ef6de384321fca6912e153b44d54 Mon Sep 17 00:00:00 2001 From: South Drifted Date: Thu, 5 Oct 2023 21:54:14 +0000 Subject: [PATCH 02/10] feat: add Object Storage configuration based on JuiceFS feat: add GitHub actions configuration of Docker Compose deployment --- .github/workflows/deploy-oss.yml | 63 ++++++++++++++++++++++++++++++++ docker-compose.yml | 13 +++++++ start-docker.sh | 9 +++++ 3 files changed, 85 insertions(+) create mode 100644 .github/workflows/deploy-oss.yml create mode 100644 start-docker.sh diff --git a/.github/workflows/deploy-oss.yml b/.github/workflows/deploy-oss.yml new file mode 100644 index 0000000..02fa95e --- /dev/null +++ b/.github/workflows/deploy-oss.yml @@ -0,0 +1,63 @@ +# reference: +# 1. https://github.com/docker/build-push-action/blob/master/docs/advanced/push-multi-registries.md +# 2. https://github.com/docker/build-push-action/blob/master/docs/advanced/share-image-jobs.md + +name: Deploy to Production environment +on: + push: + tags: + - v* +env: + ARTIFACT_PATH: /tmp/artifact.tar + BOX_NAME: api-service + BOX_URL: polyfiller/api-service + +jobs: + deploy_docker_image: + name: Deploy Docker image + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v3 + + - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-buildx-action@v2 + + - name: Build Image + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64 + tags: ${{ env.BOX_URL }}:latest + outputs: type=docker,dest=${{ env.ARTIFACT_PATH }} + + - name: Pick Docker Compose + run: | + cat > .env < Date: Fri, 6 Oct 2023 00:29:08 +0000 Subject: [PATCH 03/10] docs: add Docker deployment guide fix: some Shell Command bugs --- .github/workflows/deploy-oss.yml | 1 + Dockerfile | 1 + README.md | 57 ++++++++++++++++++++++++++++++++ start-docker.sh | 4 +-- 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-oss.yml b/.github/workflows/deploy-oss.yml index 02fa95e..cbfd406 100644 --- a/.github/workflows/deploy-oss.yml +++ b/.github/workflows/deploy-oss.yml @@ -59,5 +59,6 @@ jobs: command: | cd /tmp mv docker-compose.yml .env start-docker.sh ~/ + cd ~/ chmod +x ~/start-docker.sh sudo ~/start-docker.sh ${{ env.ARTIFACT_PATH }} diff --git a/Dockerfile b/Dockerfile index af72359..b8a07c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ COPY package.json package-lock.json file-content-injector.js /home/node/app/ RUN npm i COPY . /home/node/app +RUN npm run build RUN npm prune --production || true \ npm cache clean -f diff --git a/README.md b/README.md index d017a38..89b3172 100644 --- a/README.md +++ b/README.md @@ -362,6 +362,63 @@ These two services are very much alike. In fact, `Polyfiller` depends on the lib The server is built with support for both HTTP2 and HTTP. The environment variable `HTTP2=[true|false]` decides whether a HTTP2 server will be hosted or not. If you use a load balancer and something like `nginx` in a reverse proxy setup, please know that `nginx` doesn't support HTTP2 via its proxy module, so you have to use HTTP1.1 there. Thankfully, it is as easy as setting `HTTP2=false` before launching the server and setting `HTTPS=false`. +#### Docker + +> This guide has been tested in the deployment process of China mirror: https://polyfiller.kaiyuanshe.cn + +##### Simple container + +Run shown commands in the Project Root folder: + +```shell +docker build -t polyfiller/api-service . +docker run --name polyfiller -e NODE_ENV=production -p 3000:3000 polyfiller/api-service +``` + +##### Composed services with Object Storage + +Install Docker plugins in Cloud Server at first: + +```shell +sudo apt install docker-compose +sudo docker plugin install juicedata/juicefs +``` + +###### 1. Manual deployment + +1. Write [JuiceFS environment variables](https://juicefs.com/docs/community/juicefs_on_docker/#using-docker-compose) into `.env` file in the Project Root folder: + +```ini +STORAGE_TYPE = +BUCKET = +ACCESS_KEY = +SECRET_KEY = +``` + +2. Run shown commands in the Project Root folder: + +```shell +docker-compose up -d +``` + +###### 2. Automatic deployment + +1. Set GitHub Repository secrets: + +| name | value | +| :--------: | :-------------------------------: | +| `ENV_FILE` | `.env` file shown above | +| `HOST` | IP or Domain Name of Cloud Server | +| `USER` | Account Name of Cloud Server | +| `SSH_KEY` | SSH Private Key of Cloud Server | + +2. Push a Git tag: + +```shell +git tag v0.2.3-oss # the version number is the value of "version" field in "package.json" +git push origin --tags +``` + ## Logo All credits go to [Andreas Mehlsen (@andreasbm)](https://github.com/andreasbm/) for the awesome logo design. diff --git a/start-docker.sh b/start-docker.sh index f5051b9..7133b8b 100644 --- a/start-docker.sh +++ b/start-docker.sh @@ -3,7 +3,7 @@ docker load < $1 docker image ls -a -docker compose down -docker compose up -d +docker-compose down --volumes +docker-compose up -d docker image prune -a -f From b188437773e14871bdfcf264268de908e5e24415 Mon Sep 17 00:00:00 2001 From: South Drifted Date: Thu, 23 Nov 2023 01:05:57 +0000 Subject: [PATCH 04/10] [add] Health Check service in Docker Compose [optimize] reduce Docker image size --- Dockerfile | 24 ++++++++--------- docker-compose.yml | 66 +++++++++++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/Dockerfile b/Dockerfile index b8a07c0..a6f5a80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,20 @@ -FROM node:18-slim +# Reference: https://pnpm.io/docker#example-1-build-a-bundle-in-a-docker-container -USER root +FROM node:18-slim AS base +RUN apt-get update && \ + apt-get install curl -y --no-install-recommends +COPY . /app +WORKDIR /app -RUN npm rm yarn -g +FROM base AS prod-deps +RUN npm i --prod -RUN mkdir /home/node/app -WORKDIR /home/node/app - -COPY package.json package-lock.json file-content-injector.js /home/node/app/ +FROM base AS build RUN npm i - -COPY . /home/node/app RUN npm run build -RUN npm prune --production || true \ - npm cache clean -f - +FROM base +COPY --from=prod-deps /app/node_modules /app/node_modules +COPY --from=build /app/dist /app/dist EXPOSE 3000 CMD ["npm", "start"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 19192c0..fdb9a9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" volumes: polyfill-cache: @@ -12,28 +12,46 @@ volumes: secret-key: ${SECRET_KEY} networks: - polyfiller: + polyfiller: services: - api-service: - image: polyfiller/api-service - environment: - - NODE_ENV=production - ports: - - 3000:3000 - volumes: - - polyfill-cache:/tmp/@wessberg/polyfiller - networks: - - polyfiller - restart: always - caddy: - depends_on: - - api-service - image: caddy - ports: - - 80:80 - - 443:443 - networks: - - polyfiller - restart: always - command: caddy reverse-proxy --from polyfiller.app --to api-service:3000 + api-service: + image: polyfiller/api-service + environment: + - NODE_ENV=production + ports: + - 3000:3000 + volumes: + - polyfill-cache:/tmp/@wessberg/polyfiller + networks: + - polyfiller + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:3000/ || exit 1"] + interval: 3s + retries: 5 + start_period: 30s + labels: + - autoheal=true + restart: always + logging: + driver: json-file + options: + max-size: 10m + + autoheal: + image: willfarrell/autoheal:1.2.0 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + restart: always + + caddy: + depends_on: + - api-service + image: caddy + ports: + - 80:80 + - 443:443 + networks: + - polyfiller + restart: always + command: caddy reverse-proxy --from polyfiller.app --to api-service:3000 From 9933a77727246171a6f11b11583ff20cdb0d89af Mon Sep 17 00:00:00 2001 From: South Drifted Date: Sun, 3 Mar 2024 08:33:28 +0000 Subject: [PATCH 05/10] feat: add index.json API route --- src/api/controller/static-api-controller.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/api/controller/static-api-controller.ts b/src/api/controller/static-api-controller.ts index 450dd1c..fec95b5 100644 --- a/src/api/controller/static-api-controller.ts +++ b/src/api/controller/static-api-controller.ts @@ -7,6 +7,7 @@ import type {ApiRequest, ApiResponse} from "../server/i-server.js"; import {StatusCodes} from "http-status-codes"; import {OPTIONS} from "../decorator/api-method/options.js"; import {pickAccept} from "../util/util.js"; +import {constant} from "../../constant/constant.js"; export class StaticApiController { constructor(private readonly staticBl: IStaticBl) {} @@ -18,6 +19,17 @@ export class StaticApiController { }; } + @GET({path: "/index.json"}) + async onIndexJsonRequest({}: ApiRequest): Promise { + return { + statusCode: StatusCodes.OK, + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify(constant.polyfill) + }; + } + /** * Called when the index (/) is requested */ From cc4643ae4041f0c5d6153da28d2ea6d154a5db8e Mon Sep 17 00:00:00 2001 From: luo jiyin Date: Thu, 14 Mar 2024 18:05:14 +0800 Subject: [PATCH 06/10] [optimize] JuiceFS meta URL uses PostgreSQL (#3) Co-authored-by: South Drifted --- README.md | 7 ++++++- docker-compose.yml | 21 ++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 89b3172..db5abb4 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,11 @@ Polyfiller is kindly supported by [JetBrains](https://www.jetbrains.com/?from=Po - [FAQ](#faq) - [What's the difference from polyfill.io](#whats-the-difference-from-polyfillio) - [Hosting](#hosting) + - [Docker](#docker) + - [Simple container](#simple-container) + - [Composed services with Object Storage](#composed-services-with-object-storage) + - [1. Manual deployment](#1-manual-deployment) + - [2. Automatic deployment](#2-automatic-deployment) - [Logo](#logo) - [License](#license) - [Feature names](#feature-names) @@ -386,7 +391,7 @@ sudo docker plugin install juicedata/juicefs ###### 1. Manual deployment -1. Write [JuiceFS environment variables](https://juicefs.com/docs/community/juicefs_on_docker/#using-docker-compose) into `.env` file in the Project Root folder: +1. Write [JuiceFS object storage variables](https://juicefs.com/docs/community/reference/how_to_set_up_object_storage/) into `.env` file in the Project Root folder: ```ini STORAGE_TYPE = diff --git a/docker-compose.yml b/docker-compose.yml index fdb9a9e..c30e528 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ volumes: driver: juicedata/juicefs driver_opts: name: polyfill-cache - metaurl: sqlite3://polyfill-cache.db + metaurl: postgres://postgres:${METADATA_PASSWORD}@meta-server:5432/juicefs storage: ${STORAGE_TYPE} bucket: ${BUCKET} access-key: ${ACCESS_KEY} @@ -16,6 +16,8 @@ networks: services: api-service: + depends_on: + - meta-server image: polyfiller/api-service environment: - NODE_ENV=production @@ -37,6 +39,23 @@ services: driver: json-file options: max-size: 10m + + meta-server: + image: postgres + user: postgres + environment: + - POSTGRES_PASSWORD=${METADATA_PASSWORD} + volumes: + - ./data:/var/lib/postgresql/data/ + networks: + - polyfiller + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres" ] + interval: 3s + retries: 5 + labels: + - autoheal=true + restart: always autoheal: image: willfarrell/autoheal:1.2.0 From 023add3967b01ea253f3f9de97cf2cb195152051 Mon Sep 17 00:00:00 2001 From: South Drifted Date: Thu, 14 Mar 2024 11:10:21 +0000 Subject: [PATCH 07/10] [optimize] Docker configuration --- docker-compose.yml | 53 +++++++++++++++++++++++----------------------- package.json | 4 +++- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c30e528..1cbd53b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ volumes: driver: juicedata/juicefs driver_opts: name: polyfill-cache - metaurl: postgres://postgres:${METADATA_PASSWORD}@meta-server:5432/juicefs + metaurl: postgres://postgres:${SECRET_KEY}@meta-server:5432/postgres storage: ${STORAGE_TYPE} bucket: ${BUCKET} access-key: ${ACCESS_KEY} @@ -15,14 +15,38 @@ networks: polyfiller: services: + autoheal: + image: willfarrell/autoheal:1.2.0 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + restart: always + + meta-server: + depends_on: + - autoheal + image: postgres + user: postgres + environment: + - POSTGRES_PASSWORD=${SECRET_KEY} + volumes: + - ./data:/var/lib/postgresql/data/ + networks: + - polyfiller + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 3s + retries: 5 + labels: + - autoheal=true + restart: always + api-service: depends_on: + - autoheal - meta-server image: polyfiller/api-service environment: - NODE_ENV=production - ports: - - 3000:3000 volumes: - polyfill-cache:/tmp/@wessberg/polyfiller networks: @@ -39,29 +63,6 @@ services: driver: json-file options: max-size: 10m - - meta-server: - image: postgres - user: postgres - environment: - - POSTGRES_PASSWORD=${METADATA_PASSWORD} - volumes: - - ./data:/var/lib/postgresql/data/ - networks: - - polyfiller - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres" ] - interval: 3s - retries: 5 - labels: - - autoheal=true - restart: always - - autoheal: - image: willfarrell/autoheal:1.2.0 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - restart: always caddy: depends_on: diff --git a/package.json b/package.json index b7094b9..dcc5a64 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,9 @@ "version": "npm run generate:all && git add .", "release": "np --no-cleanup --no-yarn --no-tests", "update:check": "npx npm-check-updates -x np --dep dev,prod", - "update:commit": "npx npm-check-updates -u -x np,find-up --dep dev,prod && npm update && npm install" + "update:commit": "npx npm-check-updates -u -x np,find-up --dep dev,prod && npm update && npm install", + "pack-image": "docker build -t polyfiller/api-service:latest .", + "container": "docker rm -f api-service && docker run --name api-service -p 3000:3000 -d polyfiller/api-service:latest" }, "keywords": [ "polyfills", From faaf1cc16800773ef6dacc24402303a8a1da242b Mon Sep 17 00:00:00 2001 From: luo jiyin Date: Sat, 1 Jun 2024 07:58:29 +0800 Subject: [PATCH 08/10] Change docker db driver (#5) --- .github/workflows/deploy-oss.yml | 19 ++++++++-------- config/rclone/rclone.conf | 6 +++++ docker-compose.yml | 39 +++++++++++--------------------- start-docker.sh | 2 +- 4 files changed, 30 insertions(+), 36 deletions(-) create mode 100644 config/rclone/rclone.conf diff --git a/.github/workflows/deploy-oss.yml b/.github/workflows/deploy-oss.yml index cbfd406..9c594ec 100644 --- a/.github/workflows/deploy-oss.yml +++ b/.github/workflows/deploy-oss.yml @@ -8,7 +8,7 @@ on: tags: - v* env: - ARTIFACT_PATH: /tmp/artifact.tar + ARTIFACT_PATH: artifact.tar BOX_NAME: api-service BOX_URL: polyfiller/api-service @@ -45,20 +45,21 @@ jobs: uses: garygrossgarten/github-action-scp@release with: local: ./build - remote: /tmp + remote: /tmp/${{ env.BOX_NAME }} host: ${{ secrets.HOST }} username: ${{ secrets.USER }} - privateKey: ${{ secrets.SSH_KEY }} + password: ${{ secrets.SSH_KEY }} - name: Run Image uses: garygrossgarten/github-action-ssh@release with: host: ${{ secrets.HOST }} username: ${{ secrets.USER }} - privateKey: ${{ secrets.SSH_KEY }} + password: ${{ secrets.SSH_KEY }} command: | - cd /tmp - mv docker-compose.yml .env start-docker.sh ~/ - cd ~/ - chmod +x ~/start-docker.sh - sudo ~/start-docker.sh ${{ env.ARTIFACT_PATH }} + cd /tmp/${{ env.BOX_NAME }} + mkdir -p ~/${{ env.BOX_NAME }} + mv docker-compose.yml .env start-docker.sh ~/${{ env.BOX_NAME }} + cd ~/${{ env.BOX_NAME }} + chmod +x ./start-docker.sh + echo ${{ secrets.SSH_KEY }} | sudo -S ./start-docker.sh /tmp/${{ env.BOX_NAME }}/${{ env.ARTIFACT_PATH }} diff --git a/config/rclone/rclone.conf b/config/rclone/rclone.conf new file mode 100644 index 0000000..b8c1b93 --- /dev/null +++ b/config/rclone/rclone.conf @@ -0,0 +1,6 @@ +# Detail document in https://rclone.org/install/ & https://rclone.org/docs/ +[azure] +type = azureblob +account = xxxx +key = xxxx +endpoint = https://xxx diff --git a/docker-compose.yml b/docker-compose.yml index 1cbd53b..e8022ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,5 @@ version: "3" -volumes: - polyfill-cache: - driver: juicedata/juicefs - driver_opts: - name: polyfill-cache - metaurl: postgres://postgres:${SECRET_KEY}@meta-server:5432/postgres - storage: ${STORAGE_TYPE} - bucket: ${BUCKET} - access-key: ${ACCESS_KEY} - secret-key: ${SECRET_KEY} - networks: polyfiller: @@ -22,22 +11,20 @@ services: restart: always meta-server: - depends_on: - - autoheal - image: postgres - user: postgres + image: rclone/rclone:latest environment: - - POSTGRES_PASSWORD=${SECRET_KEY} + - RCLONE_CONFIG=/config/rclone/rclone.conf # Path to the rclone config file inside the container + cap_add: + - SYS_ADMIN # Necessary for FUSE + - MKNOD # Necessary for FUSE + devices: + - "/dev/fuse" # Necessary for FUSE + security_opt: + - "apparmor:unconfined" volumes: - - ./data:/var/lib/postgresql/data/ - networks: - - polyfiller - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 3s - retries: 5 - labels: - - autoheal=true + - ./config/rclone:/config/rclone # Mount the local rclone config directory to the container + - ./polyfill-cache:/polyfill-cache:shared # Mount the local directory for the rclone mount + command: "mount azure:/ /polyfill-cache --allow-other --allow-non-empty --vfs-cache-mode writes" restart: always api-service: @@ -48,7 +35,7 @@ services: environment: - NODE_ENV=production volumes: - - polyfill-cache:/tmp/@wessberg/polyfiller + - ./polyfill-cache:/tmp/@wessberg/polyfiller networks: - polyfiller healthcheck: diff --git a/start-docker.sh b/start-docker.sh index 7133b8b..dbd7a29 100644 --- a/start-docker.sh +++ b/start-docker.sh @@ -3,7 +3,7 @@ docker load < $1 docker image ls -a -docker-compose down --volumes +docker-compose down -v --remove-orphans docker-compose up -d docker image prune -a -f From 17ab0c085ab49a9e6390cc9934a3bc0c9a226391 Mon Sep 17 00:00:00 2001 From: luo jiyin Date: Thu, 20 Jun 2024 02:52:45 +0800 Subject: [PATCH 09/10] maintain rclone (#6) --- README.md | 100 +++++++++++++++++++++++++++++++++++++++------ docker-compose.yml | 4 +- start-docker.sh | 9 ++-- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index db5abb4..4a0d33b 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,10 @@ Polyfiller is kindly supported by [JetBrains](https://www.jetbrains.com/?from=Po - [What's the difference from polyfill.io](#whats-the-difference-from-polyfillio) - [Hosting](#hosting) - [Docker](#docker) + - [Install Docker in Cloud Server at first](#install-docker-in-cloud-server-at-first) - [Simple container](#simple-container) - [Composed services with Object Storage](#composed-services-with-object-storage) - - [1. Manual deployment](#1-manual-deployment) + - [1. Rclone config](#1-rclone-config) - [2. Automatic deployment](#2-automatic-deployment) - [Logo](#logo) - [License](#license) @@ -371,6 +372,54 @@ If you use a load balancer and something like `nginx` in a reverse proxy setup, > This guide has been tested in the deployment process of China mirror: https://polyfiller.kaiyuanshe.cn +##### Install Docker in Cloud Server at first + +In the development environment, use installation script + +```shell +curl -fsSL https://get.docker.com | sudo sh +``` + +add the current user to the `docker` user group + +```shell +sudo gpasswd -a ${USER} docker +``` + +You need to press CTRL + D to log out of the session, then use SSH to log back into the system. + +[Docker-install-script](https://github.com/docker/docker-install) says `It is not recommended to depend on this script for deployment to production systems`. +**So depend on your choice**. +Find more information in the [Docker install document](https://docs.docker.com/engine/install/). + +Rclone use `FUSE` mount the Rclone's cloud storage systems, you need to install it. + +In Debian(Ubuntu) + +```shell +sudo apt install fuse3 -y +``` + +In CentOS + +```shell +sudo yum install epel-release +sudo yum update +sudo yum install fuse3 +``` + +In Fedora + +```shell +sudo dnf install fuse3 +``` + +In Arch + +```shell +sudo pacman -Syu fuse3 +``` + ##### Simple container Run shown commands in the Project Root folder: @@ -382,28 +431,55 @@ docker run --name polyfiller -e NODE_ENV=production -p 3000:3000 polyfiller/api- ##### Composed services with Object Storage -Install Docker plugins in Cloud Server at first: +###### 1. Rclone config -```shell -sudo apt install docker-compose -sudo docker plugin install juicedata/juicefs +1. Write [Rclone config](https://rclone.org/install/) into `config/rclone/rclone.conf` file of your Project: + +```ini +[azure] +type = azureblob +account = xxxx +key = xxxx +endpoint = https://xxx ``` -###### 1. Manual deployment +How to configure quickly? Use `rclone config` command? It needs to know a lot of things. Open [Document](https://rclone.org/docs/), choose an instructions, like [Dropbox](https://rclone.org/dropbox/). -1. Write [JuiceFS object storage variables](https://juicefs.com/docs/community/reference/how_to_set_up_object_storage/) into `.env` file in the Project Root folder: +Look for ```ini -STORAGE_TYPE = -BUCKET = -ACCESS_KEY = -SECRET_KEY = +[remote] +app_key = +app_secret = +token = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXX_XXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` + +It is the configuration for Dropbox. + +```ini +[dropbox] +type = dropbox +app_key = xxxxxxxxx +app_secret = xxxxxxxxx +token = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXX_XXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` + +If you use Dropbox, in `docker-compose.yml`： + +```yaml +command: "mount azure:/ /polyfill-cache --allow-other --allow-non-empty --vfs-cache-mode writes" +``` + +It should be changed to： + +```yml +command: "mount dropbox:/ /polyfill-cache --allow-other --allow-non-empty --vfs-cache-mode writes" ``` 2. Run shown commands in the Project Root folder: ```shell -docker-compose up -d +docker compose up -d ``` ###### 2. Automatic deployment diff --git a/docker-compose.yml b/docker-compose.yml index e8022ed..5cf0c4f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock restart: always - meta-server: + polyfill-cache: image: rclone/rclone:latest environment: - RCLONE_CONFIG=/config/rclone/rclone.conf # Path to the rclone config file inside the container @@ -30,7 +30,7 @@ services: api-service: depends_on: - autoheal - - meta-server + - polyfill-cache image: polyfiller/api-service environment: - NODE_ENV=production diff --git a/start-docker.sh b/start-docker.sh index dbd7a29..4b9af70 100644 --- a/start-docker.sh +++ b/start-docker.sh @@ -1,9 +1,12 @@ #! /usr/bin/env bash -docker load < $1 +docker load <$1 docker image ls -a -docker-compose down -v --remove-orphans -docker-compose up -d +# umount polyfill-cache dir +sudo umount "$(pwd)/polyfill-cache" || true + +docker compose down -v --remove-orphans +docker compose up -d docker image prune -a -f From 023d2fdc5c1135ab95462edbbdb64f627e4428f4 Mon Sep 17 00:00:00 2001 From: luo jiyin Date: Mon, 24 Jun 2024 00:20:12 +0800 Subject: [PATCH 10/10] Use action to set rclone config (#8) --- .github/workflows/deploy-oss.yml | 5 +++-- README.md | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-oss.yml b/.github/workflows/deploy-oss.yml index 9c594ec..4065672 100644 --- a/.github/workflows/deploy-oss.yml +++ b/.github/workflows/deploy-oss.yml @@ -38,8 +38,9 @@ jobs: cat > .env < config/rclone/rclone.conf mkdir build - mv ${{ env.ARTIFACT_PATH }} docker-compose.yml .env start-docker.sh build/ + mv ${{ env.ARTIFACT_PATH }} docker-compose.yml .env start-docker.sh config/ build/ - name: Transport Image uses: garygrossgarten/github-action-scp@release @@ -59,7 +60,7 @@ jobs: command: | cd /tmp/${{ env.BOX_NAME }} mkdir -p ~/${{ env.BOX_NAME }} - mv docker-compose.yml .env start-docker.sh ~/${{ env.BOX_NAME }} + mv docker-compose.yml .env start-docker.sh config/ ~/${{ env.BOX_NAME }} cd ~/${{ env.BOX_NAME }} chmod +x ./start-docker.sh echo ${{ secrets.SSH_KEY }} | sudo -S ./start-docker.sh /tmp/${{ env.BOX_NAME }}/${{ env.ARTIFACT_PATH }} diff --git a/README.md b/README.md index 4a0d33b..ea1a617 100644 --- a/README.md +++ b/README.md @@ -486,12 +486,13 @@ docker compose up -d 1. Set GitHub Repository secrets: -| name | value | -| :--------: | :-------------------------------: | -| `ENV_FILE` | `.env` file shown above | -| `HOST` | IP or Domain Name of Cloud Server | -| `USER` | Account Name of Cloud Server | -| `SSH_KEY` | SSH Private Key of Cloud Server | +| name | value | +| :-----------: | :-------------------------------: | +| `RCLONE_FILE` | rclone configuration file | +| `ENV_FILE` | `.env` file shown above | +| `HOST` | IP or Domain Name of Cloud Server | +| `USER` | Account Name of Cloud Server | +| `SSH_KEY` | SSH Private Key of Cloud Server | 2. Push a Git tag: