Support to start vineyardd in vineyard fluid fuse. (v6d-io#1858)

Fixes v6d-io#1857

Signed-off-by: Ye Cao <[email protected]>

.github/workflows/build-vineyardd-and-wheels-linux.yaml

@@ -141,6 +141,93 @@ jobs:
  make -C docker/ vineyardd-manifest ALPINE_MANIFEST_TAG=latest
  docker manifest push ghcr.io/v6d-io/v6d/vineyardd:latest
 
+ build-vineyard-fluid-fuse:
+ needs: [build-vineyardd]
+ runs-on: ${{ matrix.os }}
+ if: ${{ github.repository == 'v6d-io/v6d' || github.event_name == 'workflow_dispatch' }}
+ strategy:
+ matrix:
+ os: [ubuntu-20.04]
+ platform: [x86_64, aarch64]
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+
+ - name: Log in to GitHub Docker Registry
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Build vineyard fluid fuse image
+ env:
+ VINEYARD_FLUID_FUSE_VERSION: "${{ github.sha }}"
+ run: |
+ make -C docker/ vineyard-fluid-fuse VINEYARD_FLUID_FUSE_VERSION=$VINEYARD_FLUID_FUSE_VERSION PLATFORM=${{ matrix.platform }}
+
+ - name: Extract tag name
+ id: tag
+ if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
+ run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+ - name: Tag and upload docker for releases
+ if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
+ run: |
+ docker tag ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ github.sha }}_${{ matrix.platform }} \
+ ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ steps.tag.outputs.TAG }}_${{ matrix.platform }}
+ docker tag ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ github.sha }}_${{ matrix.platform }} \
+ ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:latest_${{ matrix.platform }}
+
+ docker push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ github.sha }}_${{ matrix.platform }}
+ docker push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ steps.tag.outputs.TAG }}_${{ matrix.platform }}
+ docker push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:latest_${{ matrix.platform }}
+
+ build-vineyard-fluid-fuse-manifest:
+ needs: [build-vineyard-fluid-fuse]
+ runs-on: ${{ matrix.os }}
+ if: ${{ github.repository == 'v6d-io/v6d' || github.event_name == 'workflow_dispatch' }}
+ strategy:
+ matrix:
+ os: [ubuntu-20.04]
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: false
+
+ - name: Log in to GitHub Docker Registry
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Generate multi-arch manifest
+ if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'v6d-io/v6d' }}
+ run: |
+ make -C docker/ vineyard-fluid-fuse-manifest VINEYARD_FLUID_FUSE_VERSION=${{ github.sha }}
+ docker manifest push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ github.sha }}
+
+ - name: Extract tag name
+ id: tag
+ if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
+ run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+ - name: Tag and upload manifest for releases
+ if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
+ run: |
+ make -C docker/ vineyard-fluid-fuse-manifest VINEYARD_FLUID_FUSE_VERSION=${{ steps.tag.outputs.TAG }}
+ docker manifest push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ steps.tag.outputs.TAG }}
+
+ make -C docker/ vineyard-fluid-fuse-manifest VINEYARD_FLUID_FUSE_VERSION=latest
+ docker manifest push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:latest
+
  build-vineyardctl:
  runs-on: ${{ matrix.os }}
  if: ${{ github.repository == 'v6d-io/v6d' || github.event_name == 'workflow_dispatch' }}

.github/workflows/vineyard-operator.yaml

@@ -157,15 +157,6 @@ jobs:
  export IMG=ghcr.io/v6d-io/v6d/vineyard-operator:latest
  make -C k8s docker-build-push-multi-arch REGISTRY=${{ env.REGISTRY }} VERSION=latest
 
- - name: Upload latest docker image of vineyard-fluid-fuse
- if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
- run: |
- make -C docker vineyard-fluid-fuse
- docker tag ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:latest \
- ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ steps.tag.outputs.TAG }}
- docker push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:${{ steps.tag.outputs.TAG }}
- docker push ghcr.io/v6d-io/v6d/vineyard-fluid-fuse:latest
-
  - name: Generate the python and vineyardd image for tests
  run: |
  make -C k8s/test/e2e build-base-images

docker/Makefile

@@ -39,7 +39,8 @@ PYTHON_DEV_TAG := latest_$(PLATFORM)
 
 VINEYARD_FLUID_FUSE_REGISTRY := $(REGISTRY)
 VINEYARD_FLUID_FUSE_IMAGE := vineyard-fluid-fuse
-VINEYARD_FLUID_FUSE_TAG := latest
+VINEYARD_FLUID_FUSE_VERSION := latest
+VINEYARD_FLUID_FUSE_TAG := $(VINEYARD_FLUID_FUSE_VERSION)_$(PLATFORM)
 
 BUILD_PROGRESS ?= auto
 
@@ -124,5 +125,12 @@ build-python-dev:
 # build vineyard-fluid-fuse image
 vineyard-fluid-fuse:
  docker build ./vineyard-fluid-fuse \
- -t $(VINEYARD_FLUID_FUSE_REGISTRY)/$(VINEYARD_FLUID_FUSE_IMAGE):$(VINEYARD_FLUID_FUSE_TAG)
+ -t $(VINEYARD_FLUID_FUSE_REGISTRY)/$(VINEYARD_FLUID_FUSE_IMAGE):$(VINEYARD_FLUID_FUSE_TAG) \
+ --build-arg PLATFORM=$(PLATFORM)
 .PHONY: vineyard-fluid-fuse
+
+vineyard-fluid-fuse-manifest:
+ docker manifest create $(VINEYARD_FLUID_FUSE_REGISTRY)/$(VINEYARD_FLUID_FUSE_IMAGE):$(VINEYARD_FLUID_FUSE_VERSION) \
+ --amend $(VINEYARD_FLUID_FUSE_REGISTRY)/$(VINEYARD_FLUID_FUSE_IMAGE):$(VINEYARD_FLUID_FUSE_VERSION)_x86_64 \
+ --amend $(VINEYARD_FLUID_FUSE_REGISTRY)/$(VINEYARD_FLUID_FUSE_IMAGE):$(VINEYARD_FLUID_FUSE_VERSION)_aarch64
+.PHONY: vineyard-fluid-fuse-manifest

docker/vineyard-fluid-fuse/Dockerfile

@@ -1,6 +1,10 @@
-FROM busybox:1.36.1
+ARG PLATFORM=x86_64
+FROM ghcr.io/v6d-io/v6d/vineyardd:alpine-latest_x86_64 as base_x86_64
+FROM ghcr.io/v6d-io/v6d/vineyardd:alpine-latest_aarch64 as base_aarch64
+
+FROM base_$PLATFORM
 
 COPY ./entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
-CMD ["/entrypoint.sh"]
+ENTRYPOINT ["bash", "/entrypoint.sh"]

docker/vineyard-fluid-fuse/entrypoint.sh

@@ -1,17 +1,40 @@
 #!/bin/sh
 set -ex
 
-SOCKET_FILE="$FUSE_DIR/vineyard.sock"
+SOCKET_NAME="vineyard-worker.sock"
+SOCKET_FILE="$FUSE_DIR/vineyard-worker.sock"
 RPC_CONFIG_FILE="$RPC_CONF_DIR/VINEYARD_RPC_ENDPOINT"
 VINEYARD_YAML_FILE="$FUSE_DIR/vineyard-config.yaml"
 
 # Write the IPCSocket and RPCEndpoints to the vineyard configurations YAML file
 write_yaml_config() {
  echo "Vineyard:" > $VINEYARD_YAML_FILE
- echo " IPCSocket: vineyard.sock" >> $VINEYARD_YAML_FILE
+ echo " IPCSocket: $SOCKET_NAME" >> $VINEYARD_YAML_FILE
  echo " RPCEndpoint: $1" >> $VINEYARD_YAML_FILE
 }
 
+# start the standalone vineyardd
+if [ "$CACHE_SIZE" != "0" ]; then
+ vineyardd --socket=$FUSE_DIR/vineyard-local.sock \
+ --size=$CACHE_SIZE \
+ --etcd_endpoint=$ETCD_ENDPOINT \
+ --etcd_prefix=$ETCD_PREFIX &
+
+ # wait for the local vineyard socket to be created
+ timeout=60
+ count=0
+ while [ ! -S $FUSE_DIR/vineyard-local.sock ]; do
+ sleep 1
+ count=$((count+1))
+ if [ $count -eq $timeout ]; then
+ echo "Timeout waiting for $FUSE_DIR/vineyard-local.sock"
+ exit 1
+ fi
+ done
+ SOCKET_NAME="vineyard-local.sock"
+ echo "Local vineyardd is started."
+fi
+
 mkdir -p $FUSE_DIR
 while true; do
  # check if prestop marker exists, if so, skip creating the hard link
@@ -28,13 +51,33 @@ while true; do
 
  echo "write vineyard ipc socket and rpc endpoint to vineyard configuration YAML..."
  write_yaml_config "$VINEYARD_RPC_ENDPOINT"
+
  echo "check whether vineyard socket symlink is created..."
- if [ ! -S $SOCKET_FILE ] && [ -S $MOUNT_DIR/vineyard.sock ]; then
+ if [ ! -S $SOCKET_FILE ] && [ -S $MOUNT_DIR/vineyard-worker.sock ]; then
  echo "create a hard link of vineyard socket..."
- ln $MOUNT_DIR/vineyard.sock $SOCKET_FILE
+ ln $MOUNT_DIR/vineyard-worker.sock $SOCKET_FILE
  else
  echo "$SOCKET_FILE exists."
  fi
- # wait for a minute so that the hard link of vineyard socket can be checked again
- sleep 60
+
+ # avoid vineyardd restart
+ echo "check whether the inode number is same..."
+ if [ -S $SOCKET_FILE ] && [ -S $MOUNT_DIR/vineyard-worker.sock ]; then
+ SOCKET_INODE=$(ls -i $SOCKET_FILE | awk '{print $1}')
+ MOUNT_INODE=$(ls -i $MOUNT_DIR/vineyard-worker.sock | awk '{print $1}')
+ if [ "$SOCKET_INODE" != "$MOUNT_INODE" ]; then
+ echo "inode number is different, remove the hard link of vineyard socket"
+ rm -f $SOCKET_FILE
+ fi
+ fi
+
+ # avoid vineyard worker crash
+ echo "check whether vineyard worker socket exists..."
+ if [ ! -S $MOUNT_DIR/vineyard-worker.sock ]; then
+ echo "vineyard worker socket does not exist, remove the hard link of vineyard socket"
+ rm -f $SOCKET_FILE
+ fi
+
+ # wait for 5 seconds before checking again
+ sleep 5
 done