test with memgraph

dockerfile.memgraph

@@ -0,0 +1,91 @@
+FROM node:22.3.0-alpine3.20 AS node
+
+##################################### Neo4j Server #####################################
+
+FROM memgraph/memgraph:latest as memgraph
+
+###################################### Base image ######################################
+
+FROM docker:26.1.4-dind-alpine3.20 as base
+
+COPY --from=memgraph /usr/lib/memgraph/memgraph /usr/lib/memgraph/memgraph
+
+##################################### Google Cloud #####################################
+
+ARG CLOUD_SDK_VERSION=481.0.0
+ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION
+ENV PATH /google-cloud-sdk/bin:$PATH
+RUN if [ `uname -m` = 'x86_64' ]; then echo -n "x86_64" > /tmp/arch; else echo -n "arm" > /tmp/arch; fi;
+RUN ARCH=`cat /tmp/arch` && apk --no-cache add \
+        curl \
+        python3 \
+        py3-crcmod \
+        py3-openssl \
+        bash \
+        libc6-compat \
+        openssh-client \
+        git \
+        gnupg \
+    && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-${ARCH}.tar.gz && \
+    tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-${ARCH}.tar.gz && \
+    rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-${ARCH}.tar.gz && \
+    gcloud config set core/disable_usage_reporting true && \
+    gcloud config set component_manager/disable_update_check true && \
+    gcloud config set metrics/environment github_docker_image && \
+    gcloud --version
+RUN git config --system credential.'https://source.developers.google.com'.helper gcloud.sh
+
+######################################  Add node  ######################################
+
+COPY --from=node /usr/lib /usr/lib
+COPY --from=node /usr/local/lib /usr/local/lib
+COPY --from=node /usr/local/include /usr/local/include
+COPY --from=node /usr/local/bin /usr/local/bin
+
+#######################################  Script  #######################################
+
+WORKDIR /project
+
+COPY . .
+
+WORKDIR /project/sql-dump-to-neo4j
+
+RUN npm i
+
+WORKDIR /project/
+
+COPY --from=neo4j /startup/ /startup/
+
+CMD \
+  cd /project/sql-dump-to-neo4j &&\
+  # Copy the GCP credential to GCP.json file from the ENV
+  echo $GCP_JSON > GCP.json &&\
+  # Connect to Google Cloud
+  gcloud auth activate-service-account --key-file=GCP.json &&\
+  # Start Neo4j in background
+  /usr/lib/memgraph/memgraph &\
+  # Build and run the dump parser with Node js
+  npm start &&\
+  # Login to docker hub
+  docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD &&\
+  # cd to start building a image for cloud run
+  cd / &&\
+  # Build the image for the current lang
+  docker build -f /project/serverless-docker-memgraph/dockerfile.memgraph / --build-arg wiki_lang=${WIKI_LANG} -t sacramentix1225/${WIKI_LANG}wiki-memgraph &&\
+  # Push it to docker hub
+  docker push sacramentix1225/${WIKI_LANG}wiki-memgraph &&\
+  # Configure to push to GCR hub too
+  gcloud auth configure-docker europe-west9-docker.pkg.dev --quiet &&\
+  # Configure to push to GCR hub too
+  docker tag sacramentix1225/${WIKI_LANG}wiki-memgraph europe-west9-docker.pkg.dev/sixdegreesofwikiadventure/wiki-graph/${WIKI_LANG}wiki-memgraph &&\
+  # Push to GCR.io
+  docker push europe-west9-docker.pkg.dev/sixdegreesofwikiadventure/wiki-graph/${WIKI_LANG}wiki-memgraph &&\
+  # Create a new Google Cloud Run
+  gcloud run deploy ${WIKI_LANG}wiki-graph-serverless --image=europe-west9-docker.pkg.dev/sixdegreesofwikiadventure/wiki-graph/${WIKI_LANG}wiki-memgraph:latest \
+  --cpu=8 --max-instances=5 --memory=32Gi --port=8080 --allow-unauthenticated \
+  --execution-environment=gen2 \
+  --region=europe-west9 --project=sixdegreesofwikiadventure &&\
+  # Delete the Instance running this script
+  export INSTANCE_NAME=$(curl -X GET http://metadata.google.internal/computeMetadata/v1/instance/name -H 'Metadata-Flavor: Google') &&\
+  export INSTANCE_ZONE=$(curl -X GET http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google') &&\
+  gcloud --quiet compute instances delete $INSTANCE_NAME --zone=$INSTANCE_ZONE

serverless-docker-memgraph/.dockerignore

@@ -0,0 +1,5 @@
+node_modules
+build
+.env
+.env.docker
+dist

serverless-docker-memgraph/dockerfile

@@ -0,0 +1,65 @@
+FROM oven/bun:1.1.8-alpine as bun
+
+###################################### Base image ######################################
+
+ARG wiki_lang
+
+ENV ARANGO_VERSION 3.12.0.2
+ENV ARANGO_NO_AUTH 1
+
+ENV WIKI_LANG=$wiki_lang
+
+###################################### ENV & ARG  ######################################
+
+
+# see
+#   https://docs.arangodb.com/3.12/components/arangodb-server/options/#--serverendpoint
+#   https://docs.arangodb.com/3.12/components/arangodb-server/options/#log
+
+RUN apk add --no-cache gnupg pwgen binutils numactl numactl-tools && \
+    gpg --batch --keyserver keys.openpgp.org --recv-keys 8003EDF6F05459984878D4A6C04AD0FD86FEC04D && \
+    mkdir /docker-entrypoint-initdb.d && \
+    cd /tmp                                && \
+    arch="$(apk --print-arch)"             && \
+    case "$arch" in                           \
+        x86_64)  dpkgArch='amd64'          ;; \
+        aarch64) dpkgArch='arm64'          ;; \
+        *) echo >&2 "unsupported: $arch" && exit 1 ;; \
+    esac                                   && \
+    ARANGO_URL="https://download.arangodb.com/arangodb312/DEBIAN/$dpkgArch" && \
+    ARANGO_PACKAGE="arangodb3_${ARANGO_VERSION}-1_${dpkgArch}.deb" && \
+    ARANGO_PACKAGE_URL="${ARANGO_URL}/${ARANGO_PACKAGE}" && \
+    ARANGO_SIGNATURE_URL="${ARANGO_PACKAGE_URL}.asc" && \
+    wget ${ARANGO_SIGNATURE_URL}           && \
+    wget ${ARANGO_PACKAGE_URL}             && \
+    gpg --verify ${ARANGO_PACKAGE}.asc     && \
+    ar x ${ARANGO_PACKAGE} data.tar.gz     && \
+    tar -C / -x -z -f data.tar.gz          && \
+    sed -ri \
+        -e 's!127\.0\.0\.1!0.0.0.0!g' \
+        -e 's!^(file\s*=\s*).*!\1 -!' \
+        -e 's!^\s*uid\s*=.*!!' \
+        /etc/arangodb3/arangod.conf        && \
+    chgrp -R 0 /var/lib/arangodb3 /var/lib/arangodb3-apps && \
+    chmod -R 775 /var/lib/arangodb3 /var/lib/arangodb3-apps && \
+    rm -f ${ARANGO_PACKAGE}* data.tar.gz && \
+    apk del gnupg
+# Note that Openshift runs containers by default with a random UID and GID 0.
+# We need that the database and apps directory are writable for this config.
+
+ENV GLIBCXX_FORCE_NEW=1
+
+# Adjust TZ by default since tzdata package isn't present (BTS-913)
+RUN echo "UTC" > /etc/timezone
+
+COPY /var/lib/arangodb3 /var/lib/arangodb3
+
+COPY /project/serverless-docker /project
+
+WORKDIR /project
+
+RUN bun i
+
+# EXPOSE 8529 9229
+
+CMD bun index.ts

serverless-docker-memgraph/dockerfile.memgraph

@@ -0,0 +1,23 @@
+FROM node:22.3.0-alpine3.20 AS node
+
+###################################### Base image ######################################
+
+ARG wiki_lang
+
+ENV WIKI_LANG=$wiki_lang
+
+###################################### ENV & ARG  ######################################
+
+COPY /usr/lib/memgraph/memgraph /usr/lib/memgraph/memgraph
+
+COPY /var/lib/memgraph /var/lib/memgraph
+
+COPY /project/serverless-docker-memgraph /project
+
+WORKDIR /project
+
+RUN npm i
+
+RUN npm run build
+
+CMD npm start

serverless-docker-memgraph/dockerfile.neo4j

@@ -0,0 +1,62 @@
+FROM node:22.3.0-alpine3.20 AS node
+
+###################################### Base image ######################################
+
+ARG wiki_lang
+
+ENV WIKI_LANG=$wiki_lang
+
+###################################### ENV & ARG  ######################################
+
+RUN apk update
+
+RUN apk add openjdk17 su-exec
+# PATH=/var/lib/neo4j/bin:/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+ENV NEO4J_TARBALL=neo4j-community-5.20.0-unix.tar.gz \
+    NEO4J_EDITION=community \
+    NEO4J_URI=https://dist.neo4j.org/neo4j-community-5.20.0-unix.tar.gz \
+    NEO4J_HOME=/var/lib/neo4j
+
+RUN addgroup --gid 7474 --system neo4j && \
+    adduser --uid 7474 --system --no-create-home --home "${NEO4J_HOME}" --ingroup neo4j neo4j
+
+RUN apk add curl gcc git jq make procps tini wget findutils && \
+    curl --fail --silent --show-error --location --remote-name ${NEO4J_URI} && \
+    tar --extract --file ${NEO4J_TARBALL} --directory /var/lib && \
+    mv /var/lib/neo4j-* "${NEO4J_HOME}" && \
+    rm ${NEO4J_TARBALL} && \
+    mv "${NEO4J_HOME}"/data /data && \
+    mv "${NEO4J_HOME}"/logs /logs && \
+    chown -R neo4j:neo4j /data && \
+    chmod -R 777 /data && \
+    chown -R neo4j:neo4j /logs && \
+    chmod -R 777 /logs && \
+    chown -R neo4j:neo4j "${NEO4J_HOME}" && \
+    chmod -R 777 "${NEO4J_HOME}" && \
+    chmod -R 755 "${NEO4J_HOME}/bin" && \
+    ln -s /data "${NEO4J_HOME}"/data &&\
+    ln -s /logs "${NEO4J_HOME}"/logs
+
+ENV PATH="$PATH:/var/lib/neo4j/bin"
+
+WORKDIR /var/lib/neo4j/plugins
+
+RUN wget https://github.com/neo4j/apoc/releases/download/5.20.0/apoc-5.20.0-core.jar
+
+RUN mv apoc-5.20.0-core.jar apoc.jar
+
+WORKDIR /
+
+RUN echo -e "\n"dbms.security.auth_enabled=false >> "${NEO4J_HOME}"/conf/neo4j.conf
+
+COPY /data /data
+
+COPY /project/serverless-docker /project
+
+WORKDIR /project
+
+RUN npm i
+
+RUN npm run build
+
+CMD npm start

serverless-docker-memgraph/dockerfile.new

@@ -0,0 +1,10 @@
+FROM sacramentix1225/enwiki-graph
+
+COPY index.ts /project/index.ts
+
+RUN cd /project &&  npm run build
+
+RUN head -n -3 "${NEO4J_HOME}"/conf/neo4j.conf > temp && mv temp "${NEO4J_HOME}"/conf/neo4j.conf
+# RUN echo -e "\n"server.memory.heap.initial_size=16g"\n"server.memory.heap.max_size=16g"\n" >> "${NEO4J_HOME}"/conf/neo4j.conf
+
+CMD neo4j start & npm start || cat /var/lib/neo4j/logs/neo4j.log

serverless-docker-memgraph/index.ts

@@ -0,0 +1,85 @@
+// import { spawn } from "node:child_process";
+// spawn("neo4j", ["start"], { detached: true, stdio: "inherit" });
+import { Hono } from "hono";
+import { serve } from '@hono/node-server';
+import { compress } from 'hono/compress';
+import { cors } from 'hono/cors';
+import { setTimeout } from "node:timers/promises";
+import { zValidator } from '@hono/zod-validator';
+import { z } from "zod";
+import { driver } from "neo4j-driver";
+
+
+const app = new Hono();
+
+app.use('/*', cors());
+app.use(compress())
+app.notFound((c) => c.json({ message: 'Not Found', ok: false }, 404));
+app.onError((err, c) => {
+  console.error(err);
+  return c.json({ message: 'Internal Server Error', ok: false }, 500);
+});
+
+// You must listen on the port Cloud Run provides
+const port = parseInt(process.env["PORT"] || "3000");
+
+const DB_URL = "bolt://127.0.0.1:7687";
+
+const db = driver(
+    DB_URL,
+    undefined,
+    { disableLosslessIntegers: true, encrypted: false }
+);
+
+const isUp = new Promise<void>(async (res,_) => {
+    console.log(`Waiting Neo4j...`);
+    const startTime = Date.now();
+    do {    
+        try {
+            await db.getServerInfo();
+            console.log(`Neo4j up (${Date.now() - startTime}ms)`);
+            res();
+            return;
+        } catch(e) {
+        }
+        await setTimeout(50);
+    } while (true);
+});
+
+app.get(
+    "/all-shortest-path/:start/to/:end",
+    zValidator(
+        'param',
+        z.object({
+            start: z.coerce.number().int().min(1),
+            end: z.coerce.number().int().min(1),
+        })
+    ),
+    async (c, next)=>{
+        const { start, end } = c.req.param();
+        await isUp;
+        console.log(`query ${start} to ${end}`);
+        const startTime = performance.now();
+        const query = 
+`
+MATCH (from:WikiPage {id: ${start}}), (to:WikiPage {id: ${end}})
+MATCH rawPaths = (from)-[:WikiLink*BFS]->(to)
+WITH collect(DISTINCT nodes(rawPaths)) AS allPathNodes
+UNWIND allPathNodes AS pathNodes
+UNWIND pathNodes AS node
+WITH DISTINCT node, pathNodes
+WITH collect([toInteger(node.id), node.title]) AS idToTitlePairs, collect([p IN pathNodes | toInteger(p.id)]) as paths
+RETURN idToTitlePairs, paths`
+        const result = await db.session().executeWrite(tx=>tx.run(query));
+        const out = result.records[0].toObject();
+        out.time = performance.now() - startTime;
+        return c.json(out);
+});
+
+
+console.log(`Server running on port ${port}...`);
+
+serve({
+  fetch: app.fetch,
+  port,
+})