Marigold snapshot removal

charts/snapshotEngine/README.md

@@ -1,6 +1,6 @@
 # Snapshot Engine
 
-A Helm chart for creating Tezos snapshots and tarballs for faster node sync, all in kubernetes, and deploy them to a bucket with a static website.
+A Helm chart for creating Tezos snapshots and tarballs for faster node sync, all in Kubernetes, and deploy them to a bucket with a static website.
 
 Check out [xtz-shots.io](xtz-shots.io) for an example.
 
@@ -45,7 +45,7 @@ The Snapshot Engine is a Helm Chart to be deployed on a Kubernetes Cluster.  It
 
 ## How To
 
-1. Create an S3 Bucket.  
+1. Create an S3 Bucket.
 
   :warning: If you want to make it available over the internet, you will need to make it a [Public Bucket](https://aws.amazon.com/premiumsupport/knowledge-center/read-access-objects-s3-bucket/) and with the following Bucket Policy.
 
@@ -72,9 +72,9 @@ The Snapshot Engine is a Helm Chart to be deployed on a Kubernetes Cluster.  It
 
   Replace `<ARN_OF_S3_BUCKET>` with the ARN of your new S3 Bucket.
 
-  :warning: Pay close attention to the seemlingly redundant final `Resource` area. 
+  :warning: Pay close attention to the seemingly redundant final `Resource` area.
 
-  `/` and `/*` provide permission to the root and contents of the S3 Bucket respectively. 
+  `/` and `/*` provide permission to the root and contents of the S3 Bucket respectively.
 
   ```json
   {
@@ -225,7 +225,7 @@ We create only one snapshot at a time as having more than one in-progress slows
 
 ### Snapshot Scheduler Deployment
 
-A Kubernetes Deployment called the **Snapshot Scheduler** runs indefinitely triggering a new Kubernetes Job called **Snapshot Maker**.  
+A Kubernetes Deployment called the **Snapshot Scheduler** runs indefinitely triggering a new Kubernetes Job called **Snapshot Maker**.
 
 Snapshot Scheduler waits until the Snapshot Maker Job is gone to schedule a new job. This way there are snapshots constantly being created instead of running on a schedule.
 

charts/tezos/templates/_helpers.tpl

@@ -34,7 +34,7 @@
 {{- define "tezos.shouldDownloadSnapshot" -}}
   {{- if or (.Values.full_snapshot_url) (.Values.full_tarball_url)
             (.Values.rolling_snapshot_url) (.Values.rolling_tarball_url)
-            (.Values.archive_tarball_url) (.Values.snapshot_source) }}
+            (.Values.archive_tarball_url) (.Values.snapshot_source_url) (.Values.snapshot_source) }}
     {{- if or (and (.Values.rolling_tarball_url) (.Values.rolling_snapshot_url))
         (and (.Values.full_tarball_url) (.Values.full_snapshot_url))
     }}

charts/tezos/templates/configs.yaml

@@ -20,6 +20,7 @@ data:
   ARCHIVE_TARBALL_URL: "{{ .Values.archive_tarball_url }}"
   PREFER_TARBALLS: "{{ .Values.prefer_tarballs }}"
   SNAPSHOT_METADATA_NETWORK_NAME: "{{ .Values.snapshot_metadata_network_name }}"
+  SNAPSHOT_SOURCE_URL: "{{ .Values.snapshot_source_url }}"
   SNAPSHOT_SOURCE: "{{ .Values.snapshot_source }}"
   OCTEZ_VERSION: "{{ .Values.images.octez }}"
   NODE_GLOBALS: |

charts/tezos/values.yaml

@@ -316,7 +316,7 @@ serviceMonitor:
 # https://tezos.gitlab.io/user/key-management.html#signer
 octezSigners: {}
 # These signers use the octez-signer binary.
-# 
+#
 # Example:
 # ```
 # octezSigners:
@@ -427,11 +427,25 @@ dalNodes: {}
 #    storageSize: 50Gi
 #    attesterProfiles: tz1foXHgRzdYdaLgX6XhpZGxbBv42LZ6ubvE
 
-# When spinning up nodes, tezos-k8s will attempt to download a snapshot from a
-# known source. This should be a url to a json metadata file in the format
-# xtz-shots uses. If you want to sync from scratch or for a private chain, set
-# to `null`.
-snapshot_source: https://snapshots.tezos.marigold.dev/api/tezos-snapshots.json
+# When spinning up nodes, tezos-k8s attempts to download a snapshot from a
+# known source. Hard-coded snapshot URLs are used when specified.
+# NOTE: you cannot define both a `_url` and a `_tarball_url`.
+archive_tarball_url: null
+full_snapshot_url: null # e.g. https://snapshots.eu.tzinit.org/mainnet/full
+full_tarball_url: null
+rolling_snapshot_url: null # e.g. https://snapshots.us.tzinit.org/mainnet/rolling
+rolling_tarball_url: null
+
+# When a hard-coded URL is not provided, snapshots can be downloaded from
+# a URL-based source at <domain>/<network>/<mode>
+# This is the default option and set in `snapshot_source_url` for an EU region.
+# The snapshot URL for the <network> and <mode> (full or rolling) is generated.
+snapshot_source_url: https://snapshots.eu.tzinit.org/
+
+# When hard-coded URLs and a URL-based source are not provided, a URL to a
+# JSON metadata file in the xtz-shots format can be specified.
+# Set all snapshot values to `null` to sync from scratch or for a private chain.
+snapshot_source: null
 
 # By default, tezos-k8s will download and unpack snapshots.
 # A tarball is a LZ4-compressed filesystem tar of a node's data directory.
@@ -444,18 +458,6 @@ prefer_tarballs: false
 # network_name configured in `node_config_network`, hardcode the value here.
 # snapshot_metadata_network_name: "weeklynet"
 
-# By default, tezos-k8s will attempt to download the right artifact from
-# `snapshot_source` set above. You can override and hard-code a snapshot URL
-# source below. When any of the below variables are set, `snapshot_source` above
-# will be ignored for all artifact types.
-## NOTE: `*_tarball_url` and `*_snapshot_url` are mutually exclusive
-## and cannot both be specified at the same time.
-archive_tarball_url: null # e.g. https://mainnet.xtz-shots.io/archive-tarball
-full_snapshot_url: null
-full_tarball_url: null
-rolling_snapshot_url: null
-rolling_tarball_url: null
-
 # List of peers for nodes to connect to. Gets set under config.json `p2p` field
 bootstrap_peers: []
 

mkchain/tqchain/mkchain.py

@@ -156,6 +156,7 @@ def main():
         },
         "node_config_network": {"chain_name": args.chain_name},
         # Custom chains should not pull snapshots or tarballs
+        "snapshot_source_url": None,
         "snapshot_source": None,
         "node_globals": {
             # Needs a quotedstring otherwise helm interprets "Y" as true and it does not work

test/charts/mainnet.expect.yaml

@@ -34,7 +34,8 @@ data:
   ARCHIVE_TARBALL_URL: ""
   PREFER_TARBALLS: "false"
   SNAPSHOT_METADATA_NETWORK_NAME: ""
-  SNAPSHOT_SOURCE: "https://snapshots.tezos.marigold.dev/api/tezos-snapshots.json"
+  SNAPSHOT_SOURCE_URL: "https://snapshots.eu.tzinit.org/"
+  SNAPSHOT_SOURCE: ""
   OCTEZ_VERSION: "tezos/tezos:v19.0"
   NODE_GLOBALS: |
     {

test/charts/mainnet2.expect.yaml

@@ -34,7 +34,8 @@ data:
   ARCHIVE_TARBALL_URL: ""
   PREFER_TARBALLS: "false"
   SNAPSHOT_METADATA_NETWORK_NAME: ""
-  SNAPSHOT_SOURCE: "https://snapshots.tezos.marigold.dev/api/tezos-snapshots.json"
+  SNAPSHOT_SOURCE_URL: "https://snapshots.eu.tzinit.org/"
+  SNAPSHOT_SOURCE: ""
   OCTEZ_VERSION: "tezos/tezos:v19.0"
   NODE_GLOBALS: |
     {

test/charts/private-chain.expect.yaml

@@ -127,6 +127,7 @@ data:
   ARCHIVE_TARBALL_URL: ""
   PREFER_TARBALLS: "false"
   SNAPSHOT_METADATA_NETWORK_NAME: ""
+  SNAPSHOT_SOURCE_URL: ""
   SNAPSHOT_SOURCE: ""
   OCTEZ_VERSION: "tezos/tezos:v15-release"
   NODE_GLOBALS: |

test/charts/private-chain.in.yaml

@@ -72,6 +72,7 @@ activation:
     sc_rollup_max_available_messages: 1000000
 bootstrap_peers: []
 expected_proof_of_work: 0
+snapshot_source_url: null
 snapshot_source: null
 images:
   octez: 'tezos/tezos:v15-release'

utils/config-generator.py

@@ -144,6 +144,10 @@ def main():
                 node_snapshot_config,
                 indent=2,
             )
+
+            print("Generated node_snapshot_config_json :")
+            print(node_snapshot_config_json)
+
             if node_snapshot_config:
                 print("Generated snapshot_config.json :")
                 print(node_snapshot_config_json)
@@ -582,12 +586,14 @@ def create_node_snapshot_config_json(history_mode):
     archive_tarball_url = os.environ.get("ARCHIVE_TARBALL_URL")
     rolling_snapshot_url = os.environ.get("ROLLING_SNAPSHOT_URL")
     full_snapshot_url = os.environ.get("FULL_SNAPSHOT_URL")
+    snaphot_source_url = os.environ.get("SNAPSHOT_SOURCE_URL")
     if (
         rolling_tarball_url
         or full_tarball_url
         or rolling_snapshot_url
         or full_snapshot_url
         or archive_tarball_url
+        or snaphot_source_url
     ):
         print("Snapshot or tarball URL found, will ignore snapshot_source")
         match history_mode:
@@ -599,16 +605,31 @@ def create_node_snapshot_config_json(history_mode):
                         "url": rolling_snapshot_url,
                         "artifact_type": "tezos-snapshot",
                     }
+                elif snaphot_source_url:
+                    return {
+                        "url": make_url(snaphot_source_url, network_name, history_mode),
+                        "artifact_type": "tezos-snapshot",
+                    }
                 return
             case "full":
                 if full_tarball_url:
                     return {"url": full_tarball_url, "artifact_type": "tarball"}
                 elif full_snapshot_url:
                     return {"url": full_snapshot_url, "artifact_type": "tezos-snapshot"}
+                elif snaphot_source_url:
+                    return {
+                        "url": make_url(snaphot_source_url, network_name, history_mode),
+                        "artifact_type": "tezos-snapshot",
+                    }
                 return
             case "archive":
                 if archive_tarball_url:
                     return {"url": archive_tarball_url, "artifact_type": "tarball"}
+                elif snaphot_source_url:
+                    return {
+                        "url": make_url(snaphot_source_url, network_name, history_mode + '.tar.lz4'),
+                        "artifact_type": "tarball",
+                    }
                 return
             case _:
                 print(f"Error: history mode {history_mode} is not known.")
@@ -670,5 +691,13 @@ def create_node_snapshot_config_json(history_mode):
     return matching_snapshots[-1] if len(matching_snapshots) else None
 
 
+def make_url(domain, *path):
+    url = domain.rstrip('/')
+    for p in path:
+        _p = p.strip('/')
+        url = '{}/{}'.format(url, _p) if _p else url
+    return url
+
+
 if __name__ == "__main__":
     main()