Add unmanaged_tablet_test.sh to run a new e2e build on CI

.buildkite/pipeline.yml

@@ -82,3 +82,22 @@ steps:
           propagate-environment: true
           volumes:
             - "/var/run/docker.sock:/var/run/docker.sock"
+
+  - name: "Unmanaged Tablet Test"
+    command:
+    - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat chromium
+    - wget https://golang.org/dl/$GO_VERSION_FILE
+    - tar -C /usr/local -xzf $GO_VERSION_FILE
+    - export PATH=$PATH:/usr/local/go/bin
+    - rm $GO_VERSION_FILE
+    - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2
+    - make unmanaged-tablet-test
+    concurrency: 1
+    concurrency_group: 'vtop/unmanaged-tablet-test'
+    timeout_in_minutes: 60
+    plugins:
+      - docker#v3.12.0:
+          image: "docker:latest"
+          propagate-environment: true
+          volumes:
+            - "/var/run/docker.sock:/var/run/docker.sock"

Makefile

@@ -73,3 +73,7 @@ backup-schedule-test: build e2e-test-setup
 vtorc-vtadmin-test: build e2e-test-setup
 	echo "Running VTOrc and VtAdmin test"
 	test/endtoend/vtorc_vtadmin_test.sh
+
+unmanaged-tablet-test: build e2e-test-setup
+	echo "Running Unmanaged Tablet test"
+	test/endtoend/unmanaged_tablet_test.sh

docs/release-process.md

@@ -107,7 +107,7 @@ from the pull request, so in order to have the tag on the release branche's hist
 
 ##### Update the test output
 
-The `upgrade_test.sh`, `backup_restore_test.sh` and `vtorc_vtadmin_test.sh` files must be updated with the proper release increment. Change the `verifyVtGateVersion` function calls to use the proper version (current version being released and latest previous version (only used in `upgrade_test.sh`)).
+The `upgrade_test.sh`, `backup_restore_test.sh`, `vtorc_vtadmin_test.sh` and `unmanaged_tablet_test.sh` files must be updated with the proper release increment. Change the `verifyVtGateVersion` function calls to use the proper version (current version being released and latest previous version (only used in `upgrade_test.sh`)).
 
 ##### CI Failures
 

test/endtoend/operator/101_initial_cluster_unmanaged_tablet.yaml

@@ -0,0 +1,259 @@
+# The following example is minimalist. The security policies
+# and resource specifications are not meant to be used in production.
+# Please refer to the operator documentation for recommendations on
+# production settings.
+apiVersion: planetscale.com/v2
+kind: VitessCluster
+metadata:
+  name: example
+spec:
+  images:
+    vtctld: vitess/lite:latest
+    vtadmin: vitess/vtadmin:latest
+    vtgate: vitess/lite:latest
+    vttablet: vitess/lite:latest
+    vtbackup: vitess/lite:latest
+    vtorc: vitess/lite:latest
+    mysqld:
+      mysql80Compatible: mysql:8.0.30
+    mysqldExporter: prom/mysqld-exporter:v0.11.0
+  cells:
+  - name: zone1
+    gateway:
+      authentication:
+        static:
+          secret:
+            name: example-cluster-config
+            key: users.json
+      replicas: 1
+      resources:
+        requests:
+          cpu: 100m
+          memory: 256Mi
+        limits:
+          memory: 256Mi
+  vitessDashboard:
+    cells:
+    - zone1
+    extraFlags:
+      security_policy: read-only
+    replicas: 1
+    resources:
+      limits:
+        memory: 128Mi
+      requests:
+        cpu: 100m
+        memory: 128Mi
+
+  keyspaces:
+  - name: commerce
+    durabilityPolicy: none
+    turndownPolicy: Immediate
+    vitessOrchestrator:
+      resources:
+        limits:
+          memory: 128Mi
+        requests:
+          cpu: 10m
+          memory: 64Mi
+      extraFlags:
+        recovery-period-block-duration: 5s
+    partitionings:
+    - equal:
+        parts: 1
+        shardTemplate:
+          databaseInitScriptSecret:
+            name: example-cluster-config
+            key: init_db.sql
+          tabletPools:
+          - cell: zone1
+            type: externalmaster
+            replicas: 1
+            vttablet:
+              extraFlags:
+                db_charset: utf8mb4
+              resources:
+                limits:
+                  memory: 256Mi
+                requests:
+                  cpu: 100m
+                  memory: 256Mi
+            externalDatastore:
+              user: root
+              host: mysql
+              port: 3306
+              database: main
+              credentialsSecret:
+                name: example-cluster-config
+                key: ext_db_credentials_secret.json
+  updateStrategy:
+    type: Immediate
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: example-cluster-config
+type: Opaque
+stringData:
+  users.json: |
+    {
+      "user": [{
+        "UserData": "user",
+        "Password": ""
+      }]
+    }
+  init_db.sql: |
+    # This file is executed immediately after mysql_install_db,
+    # to initialize a fresh data directory.
+
+    ###############################################################################
+    # Equivalent of mysql_secure_installation
+    ###############################################################################
+    # We need to ensure that super_read_only is disabled so that we can execute
+    # these commands. Note that disabling it does NOT disable read_only.
+    # We save the current value so that we only re-enable it at the end if it was
+    # enabled before.
+    SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');
+    SET GLOBAL super_read_only='OFF';
+
+    # Changes during the init db should not make it to the binlog.
+    # They could potentially create errant transactions on replicas.
+    SET sql_log_bin = 0;
+    # Remove anonymous users.
+    DELETE FROM mysql.user WHERE User = '';
+
+    # Disable remote root access (only allow UNIX socket).
+    DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+
+    # Remove test database.
+    DROP DATABASE IF EXISTS test;
+
+    ###############################################################################
+    # Vitess defaults
+    ###############################################################################
+
+    # Vitess-internal database.
+    CREATE DATABASE IF NOT EXISTS _vt;
+    # Note that definitions of local_metadata and shard_metadata should be the same
+    # as in production which is defined in go/vt/mysqlctl/metadata_tables.go.
+    CREATE TABLE IF NOT EXISTS _vt.local_metadata (
+      name VARCHAR(255) NOT NULL,
+      value VARCHAR(255) NOT NULL,
+      db_name VARBINARY(255) NOT NULL,
+      PRIMARY KEY (db_name, name)
+      ) ENGINE=InnoDB;
+    CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
+      name VARCHAR(255) NOT NULL,
+      value MEDIUMBLOB NOT NULL,
+      db_name VARBINARY(255) NOT NULL,
+      PRIMARY KEY (db_name, name)
+      ) ENGINE=InnoDB;
+
+    # Admin user with all privileges.
+    CREATE USER 'vt_dba'@'localhost';
+    GRANT ALL ON *.* TO 'vt_dba'@'localhost';
+    GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';
+
+    # User for app traffic, with global read-write access.
+    CREATE USER 'vt_app'@'localhost';
+    GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+      REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+      LOCK TABLES, EXECUTE, REPLICATION CLIENT, CREATE VIEW,
+      SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+      ON *.* TO 'vt_app'@'localhost';
+
+    # User for app debug traffic, with global read access.
+    CREATE USER 'vt_appdebug'@'localhost';
+    GRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';
+
+    # User for administrative operations that need to be executed as non-SUPER.
+    # Same permissions as vt_app here.
+    CREATE USER 'vt_allprivs'@'localhost';
+    GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+      REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+      LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+      SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+      ON *.* TO 'vt_allprivs'@'localhost';
+
+    # User for slave replication connections.
+    # TODO: Should we set a password on this since it allows remote connections?
+    CREATE USER 'vt_repl'@'%';
+    GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';
+
+    # User for Vitess filtered replication (binlog player).
+    # Same permissions as vt_app.
+    CREATE USER 'vt_filtered'@'localhost';
+    GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+      REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+      LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+      SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+      ON *.* TO 'vt_filtered'@'localhost';
+
+    FLUSH PRIVILEGES;
+
+    RESET SLAVE ALL;
+    RESET MASTER;
+
+    # custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
+    # {{custom_sql}}
+
+    # We need to set super_read_only back to what it was before
+    SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'ON');
+  ext_db_credentials_secret.json: |
+    {
+      "root": ["password"]
+    }
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: mysql-pv-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  ports:
+    - port: 3306
+  selector:
+    app: mysql
+  clusterIP: None
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mysql
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      containers:
+        - name: mysql
+          image: mysql:8.0
+          env:
+            - name: MYSQL_ROOT_PASSWORD
+              value: password
+          ports:
+            - containerPort: 3306
+          volumeMounts:
+            - name: mysql-persistent-storage
+              mountPath: /var/lib/mysql
+      volumes:
+        - name: mysql-persistent-storage
+          persistentVolumeClaim:
+            claimName: mysql-pv-claim