{Earthfile, Jenkinsfile}: Earthly ci testing

Change-Id: Idc1292cddab20526afa2e3f783dd4b1f51891efd
storj · Oct 11, 2023 · 1dc4e76 · 1dc4e76
1 parent 71471d6
commit 1dc4e76
Show file tree

Hide file tree

Showing 2 changed files with 90 additions and 67 deletions.
diff --git a/Earthfile b/Earthfile
@@ -18,7 +18,19 @@ lint:
     RUN check-monkit ./...
     RUN check-mod-tidy
 
+deps:
+    # Download deps before copying code.
+    COPY go.mod go.sum ./pkg/config/gen/go.mod ./pkg/config/gen/go.sum .
+    RUN go mod download
+    # Output these back in case go mod download changes them.
+    SAVE ARTIFACT go.mod AS LOCAL go.mod
+    SAVE ARTIFACT go.sum AS LOCAL go.sum
+    SAVE ARTIFACT ./pkg/config/gen/go.mod AS LOCAL ./pkg/config/gen/go.mod
+    SAVE ARTIFACT ./pkg/config/gen/go.sum AS LOCAL ./pkg/config/gen/go.sum
+
 build:
+    FROM +deps
+    # Copy and build code.
     COPY . .
     RUN --mount=type=cache,target=/root/.cache/go-build \
         --mount=type=cache,target=/go/pkg/mod \
@@ -40,56 +52,58 @@ test:
        ./scripts/test.sh
    SAVE ARTIFACT build/tests.json AS LOCAL build/tests.json
 
-step1:
-    LOCALLY
-    RUN whoami
-    RUN cat /proc/1/cgroup | head -n 1
-
-step2:
-    FROM alpine:latest
-    RUN whoami
-    RUN cat /proc/1/cgroup | head -n 1
+integration-uplink:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build/storj-up .
+    RUN /go/storj-up/storj-up init minimal,db,uplink
+    COPY ./test/uplink/basic_upload_download.sh basic_upload_download.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' && \
+            < basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i
+    END
 
-both:
-    BUILD +step1
-    BUILD +step2
+integration-edge:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build/storj-up .
+    RUN /go/storj-up/storj-up init minimal,edge,db,uplink && \
+        /go/storj-up/storj-up image uplink img.dev.storj.io/dlm/storj:1 && \
+        /go/storj-up/storj-up env set authservice STORJ_ENDPOINT=http://gateway-mt:9999
+    COPY ./test/edge/basic_rclone.sh basic_rclone.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e --s3 >> ~/.bashrc' && \
+            < basic_rclone.sh docker compose exec -T -u 0 uplink bash -i
+    END
 
-ci-integration:
-   FROM earthly/dind:ubuntu
-   RUN apt-get update && apt-get install -y golang-go docker-compose-plugin gcc
-   ENV PATH=$PATH:/root/go/bin
-   COPY . .
-   WORKDIR /test
-   WITH DOCKER --pull img.dev.storj.io/storjup/storj:1.83.2 \
-               --pull img.dev.storj.io/dlm/storj:1 \
-               --pull img.dev.storj.io/storjup/edge:1.58.1 \
-               --pull cockroachdb/cockroach \
-               --pull redis:6.0.9 \
-               --pull img.dev.storj.io/storjup/storjscan:1.0.3 \
-               --pull img.dev.storj.io/dlm/storjscan:1 \
-               --pull ethereum/client-go
-      RUN /test/uplink/storj-up.sh && \
-          /test/edge/storj-up.sh && \
-          /test/storjscan/storj-up.sh
-   END
+integration-storjscan:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build/storj-up .
+    RUN /go/storj-up/storj-up init && \
+        /go/storj-up/storj-up init minimal,satellite-core,edge,db,billing && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_BILLING_CONFIG_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_CONFIRMATIONS=12 && \
+        /go/storj-up/storj-up env setenv storjscan STORJ_TOKEN_PRICE_USE_TEST_PRICES=true && \
+        /go/storj-up/storj-up image storjscan img.dev.storj.io/dlm/storjscan:1
+    COPY ./test/storjscan/basic_storjscan.sh basic_storjscan.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 storjscan bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 storjscan bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e >> ~/.bashrc' && \
+            docker compose cp pk.json storjscan:/var/lib/storj/pk.json && \
+            docker compose cp pass storjscan:/var/lib/storj/pass && \
+            docker compose cp TestToken.abi storjscan:/var/lib/storj/TestToken.abi && \
+            docker compose cp TestToken.bin storjscan:/var/lib/storj/TestToken.bin && \
+            < basic_storjscan.sh docker compose exec -T -u 0 storjscan bash -i
+    END
 
-local-integration:
-#   FROM earthly/dind:ubuntu
-#   RUN apt-get update && apt-get install -y golang-go docker-compose-plugin gcc
-#   ENV PATH=$PATH:/root/go/bin
-#   COPY . .
- #  WORKDIR /test
-   LOCALLY
-   WITH DOCKER --pull img.dev.storj.io/storjup/storj:1.83.2 \
-               --pull img.dev.storj.io/storjup/edge:1.58.1 \
-               --pull cockroachdb/cockroach \
-               --pull redis:6.0.9 \
-               --pull img.dev.storj.io/storjup/storjscan:1.0.3 \
-               --pull ethereum/client-go
-      RUN /test/uplink/storj-up.sh && \
-          /test/edge/storj-up.sh && \
-          /test/storjscan/storj-up.sh
-   END
+integration-all:
+   BUILD +integration-uplink
+   BUILD +integration-edge
+   BUILD +integration-storjscan
 
 check-format:
    COPY . .

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -17,25 +17,34 @@ pipeline {
                checkout scm
             }
         }
-	stage('Lint') {
-		steps {
-			sh "earthly +lint"
-		}
-	}
-	stage('Test') {
-		steps {
-			sh "earthly +test"
-		}
-	}
-	stage('Integration') {
-		steps {
-			withCredentials([usernamePassword(credentialsId: '67d52f80-1802-407a-a21a-ea24d4c28a83', passwordVariable: 'password', usernameVariable: 'username')]) {
-				sh "docker login img.dev.storj.io -u $username -p $password"
-                                sh "docker pull img.dev.storj.io/dlm/storj:1"
-                                sh "docker pull img.dev.storj.io/dlm/storjscan:1"
-				sh "earthly -P +ci-integration"
-			}
-		}
-	}
+        stage('Lint') {
+            steps {
+                sh "earthly +lint"
+            }
+        }
+        stage('Test') {
+            steps {
+                sh "earthly +test"
+            }
+        }
+        stage('Integration') {
+            parallel {
+                stage('Uplink') {
+                    steps {
+                        sh "earthly -P +integration-uplink"
+                    }
+                }
+                stage('Edge') {
+                    steps {
+                        sh "earthly -P +integration-edge"
+                    }
+                }
+                stage('Storjscan') {
+                    steps {
+                        sh "earthly -P +integration-storjscan"
+                    }
+                }
+            }
+        }
     }
 }