Merge pull request #702 from CodeForAfrica/ft/robots-generator

Robots Generator

.github/workflows/roboshield-deploy-dev.yml

@@ -0,0 +1,76 @@
+name: RoboShield | Deploy | DEV
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "apps/roboshield/**"
+      - "Dockerfile.roboshield"
+      - ".github/workflows/roboshield-dev.yml"
+
+concurrency:
+  group: "${{ github.workflow }} @ ${{ github.ref }}"
+  cancel-in-progress: true
+
+env:
+  DOKKU_REMOTE_BRANCH: "master"
+  DOKKU_REMOTE_URL: "ssh://[email protected]"
+  IMAGE_NAME: "codeforafrica/roboshield"
+  NEXT_PUBLIC_APP_URL: "https://roboshield-ui.dev.codeforafrica.org"
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  APP_NAME: roboshield-ui
+
+jobs:
+  deploy:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node-version: [20]
+        os: [ubuntu-latest]
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          path: /tmp/.buildx-cache
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v3
+        with:
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+          context: .
+          file: Dockerfile.roboshield
+          push: true
+          tags: "${{ env.IMAGE_NAME }}:${{ github.sha }}"
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Push to Dokku
+        uses: dokku/[email protected]
+        with:
+          git_remote_url: ${{ env.DOKKU_REMOTE_URL }}/${{ env.APP_NAME }}
+          ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
+          deploy_docker_image: ${{ env.IMAGE_NAME }}:${{ github.sha }}

.gitignore

@@ -60,3 +60,7 @@ credentials.json
 
 # Sentry Config File
 .sentryclirc
+
+# SQLite files
+**.db
+**.sqlite

Dockerfile.roboshield

@@ -0,0 +1,116 @@
+FROM node:20.14-alpine as node
+
+# Always install security updated e.g. https://pythonspeed.com/articles/security-updates-in-docker/
+# Update local cache so that other stages don't need to update cache
+RUN apk update \
+    && apk upgrade
+
+# ===================================================
+# base: starting image to be used in all other stages
+# ===================================================
+FROM node as base
+
+# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
+RUN apk add --no-cache libc6-compat
+
+WORKDIR /workspace
+
+
+# ===================================================
+# pnpm-base: starting image with pnpm activated.
+#            should be used in all "build" stages.
+# ===================================================
+FROM base as pnpm-base
+
+ARG PNPM_VERSION=9.1.4
+
+RUN corepack enable && corepack prepare pnpm@${PNPM_VERSION} --activate
+
+# =========================================================
+# desp: image with all dependencies (for the app) installed
+# =========================================================
+FROM pnpm-base as deps
+
+COPY pnpm-lock.yaml .
+
+RUN pnpm fetch
+
+COPY *.yaml *.json ./
+COPY packages/commons-ui-core/package.json ./packages/commons-ui-core/package.json
+COPY packages/commons-ui-next/package.json ./packages/commons-ui-next/package.json
+COPY apps/roboshield/package.json ./apps/roboshield/package.json
+
+# Use virtual store: https://pnpm.io/cli/fetch#usage-scenario
+RUN pnpm --filter "./apps/roboshield" install --offline --frozen-lockfile
+
+
+# =======================================================
+# builder: image that uses deps to build shippable output
+# =======================================================
+FROM pnpm-base as builder
+
+ARG NEXT_TELEMETRY_DISABLED=1 \
+    # Needed by Next.js at build time
+    NEXT_PUBLIC_APP_NAME=RoboShield \
+    NEXT_PUBLIC_APP_URL=http://localhost:3000 \
+    NEXT_PUBLIC_SEO_DISABLED="true" \
+    # Needed by Next.js and server.ts at build time
+    PORT=3000
+
+COPY --from=deps /workspace/node_modules ./node_modules
+COPY --from=deps /workspace/packages/commons-ui-core/node_modules ./packages/commons-ui-core/node_modules
+COPY --from=deps /workspace/packages/commons-ui-next/node_modules ./packages/commons-ui-next/node_modules
+COPY --from=deps /workspace/apps/roboshield/node_modules ./apps/roboshield/node_modules
+
+COPY packages/commons-ui-core ./packages/commons-ui-core
+COPY packages/commons-ui-next ./packages/commons-ui-next
+COPY apps/roboshield ./apps/roboshield
+
+RUN pnpm --filter "./apps/roboshield" build
+
+
+# ==============================
+# runner: final deployable image
+# ==============================
+FROM base as runner
+
+ARG NEXT_TELEMETRY_DISABLED \
+    NEXT_PUBLIC_APP_NAME \
+    NEXT_PUBLIC_APP_URL \
+    PORT
+
+ENV NODE_ENV=production \
+    NEXT_PUBLIC_APP_NAME=${NEXT_PUBLIC_APP_NAME} \
+    NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL} \
+    PORT=${PORT}
+
+RUN set -ex \
+    # Create a non-root user
+    && addgroup --system -g 1001 nodejs \
+    && adduser --system -u 1001 -g 1001 nextjs \
+    # Create nextjs cache dir w/ correct permissions
+    && mkdir -p ./apps/roboshield/.next \
+    && chown nextjs:nodejs ./apps/roboshield/.next \
+    # Delete system cached files we don't need anymore
+    && rm -rf /var/cache/apk/*
+
+# PNPM symlink some dependencies
+COPY --from=builder --chown=nextjs:nodejs /workspace/node_modules ./node_modules
+# Public assets
+COPY --from=builder --chown=nextjs:nodejs /workspace/apps/roboshield/public ./apps/roboshield/public
+
+# Automatically leverage output traces to reduce image size
+# https://nextjs.org/docs/advanced-features/output-file-tracing
+COPY --from=builder --chown=nextjs:nodejs /workspace/apps/roboshield/.next/standalone ./apps/roboshield
+COPY --from=builder --chown=nextjs:nodejs /workspace/apps/roboshield/.next/static ./apps/roboshield/.next/static
+
+USER nextjs
+
+EXPOSE ${PORT}
+
+# set hostname to localhost
+ENV HOSTNAME "0.0.0.0"
+
+# server.js is created by next build from the standalone output
+# https://nextjs.org/docs/pages/api-reference/next-config-js/output
+CMD ["node", "apps/roboshield/server.js"]

Makefile

@@ -18,5 +18,9 @@ mongodb-keyfile:
 	openssl rand -base64 741 > ./mongo-keyfile
 	chmod 600 ./mongo-keyfile
 
+
 pesayetu:
 	$(COMPOSE_BUILD_ENV) $(COMPOSE) --env-file apps/pesayetu/.env.local up pesayetu --build
+
+roboshield:
+	$(COMPOSE_BUILD_ENV) $(COMPOSE) --env-file apps/roboshield/.env.local up roboshield --build

README.md

@@ -7,6 +7,7 @@
 | [**charterAFRICA**](./apps/charterafrica/)   | The largest digital database for communities              |
 | [**Code for Africa**](./apps/codeforafrica/) | Africa's largest network of civic tech and open data labs |
 | [**PesaYetu**](./apps/pesayetu/)             | Data to hold your government accountable                  |
+| [**RoboShield**](./apps/roboshield/)         | Guard your website against AI Bots                        |
 
 ## Get started
 

apps/roboshield/.eslintignore

@@ -0,0 +1,36 @@
+# dependencies
+node_modules
+.pnp
+.pnp.js
+.pnpm-debug.log
+
+# typescript
+dist/
+
+# testing
+coverage
+
+# next.js
+.next/
+out/
+
+# payload
+build/
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Vercel
+.vercel
+.now
+
+# turbo
+.turbo
+test-results/
+playwright-report/

apps/roboshield/.eslintrc.js

@@ -0,0 +1,11 @@
+module.exports = {
+  root: true,
+  extends: ["next/core-web-vitals", "plugin:prettier/recommended"],
+  settings: {
+    "import/resolver": {
+      webpack: {
+        config: "./eslint.webpack.config.js",
+      },
+    },
+  },
+};

apps/roboshield/.lintstagedrc.js

@@ -0,0 +1,14 @@
+const path = require("path");
+
+const buildEslintCommand = (filenames) =>
+  `next lint --fix --file ${filenames
+    .map((f) => path.relative(process.cwd(), f))
+    .join(" --file ")}`;
+
+module.exports = {
+  // Since we don't have eslint json/md plugins installed in this app, we can't
+  // use the eslint to lint json,md here
+  "*.{json,md}": ["prettier --write"],
+  "*.{yaml,yml}": "prettier --write",
+  "*.{js,mjs,cjs,jsx,ts,mts,cts,tsx}": [buildEslintCommand],
+};

apps/roboshield/README.md

@@ -0,0 +1,17 @@
+RoboShield is a web application that allows users to create and manage their `robots.txt` files.
+
+## Getting Started
+
+### Install dependencies
+
+```bash
+pnpm install
+```
+
+First, run the development server:
+
+```bash
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.

apps/roboshield/eslint.webpack.config.js

@@ -0,0 +1,25 @@
+const path = require("path");
+
+module.exports = {
+  module: {
+    rules: [
+      {
+        test: /\.svg$/i,
+        type: "asset",
+        resourceQuery: /url/, // *.svg?url
+      },
+      {
+        test: /\.svg$/i,
+        issuer: /\.[jt]sx?$/,
+        resourceQuery: { not: [/url/] }, // exclude react component if *.svg?url
+        use: ["@svgr/webpack"],
+      },
+    ],
+  },
+  resolve: {
+    alias: {
+      "@/roboshield": path.resolve(__dirname, "src/"),
+    },
+    extensions: [".ts", ".tsx"],
+  },
+};

apps/roboshield/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/basic-features/typescript for more information.

apps/roboshield/next.config.mjs

@@ -0,0 +1,35 @@
+import path from "path";
+
+const PROJECT_ROOT = process.env.PROJECT_ROOT?.trim();
+const outputFileTracingRoot = PROJECT_ROOT
+  ? path.resolve(__dirname, PROJECT_ROOT)
+  : undefined;
+
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  transpilePackages: ["@commons-ui/core", "@commons-ui/next"],
+  reactStrictMode: true,
+  output: "standalone",
+  experimental: {
+    outputFileTracingRoot,
+  },
+  webpack: (config) => {
+    config.module.rules.push(
+      {
+        test: /\.svg$/i,
+        type: "asset",
+        resourceQuery: /url/, // *.svg?url
+      },
+      {
+        test: /\.svg$/i,
+        issuer: /\.[jt]sx?$/,
+        resourceQuery: { not: [/url/] }, // exclude react component if *.svg?url
+        use: ["@svgr/webpack"],
+      },
+    );
+    config.experiments = { ...config.experiments, topLevelAwait: true }; // eslint-disable-line no-param-reassign
+    return config;
+  },
+};
+
+export default nextConfig;