clusterversion: create 25.1 version

This change bumps the version on master to 25.1 without advancing the
previous release to 24.3 (the latter can only be done once we have a
published RC, or upgrade tests will fail). To allow this decoupling, a
few minor tweaks were needed (to `SupportedPreviousReleases` and to
the tool that generates the releases file.

Also made some minor improvements to the runbook and
sql-bootstrap-data. I will do a more thorough change to the runbook
that separates this step.

- [x] Add version key constant for new release, equal to `Latest`
- [x] Update `PreviousRelease` constant
- [x] Add start version
- [x] Update `pkg/build/version.txt` to the new version
- [x] Add mixed version logictest config for the replaced version
- [x] Update the `scplan` rules in `pkg/sql/schemachanger/scplan/internal/rules` (already done)
- [x] Create new SQL bootstrap data
- [x] Update releases file

Epic: REL-1322

docs/generated/settings/settings-for-tenants.txt

@@ -402,4 +402,4 @@ trace.span_registry.enabled	boolean	true	if set, ongoing traces can be seen at h
 trace.zipkin.collector	string		the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.	application
 ui.database_locality_metadata.enabled	boolean	true	if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute	application
 ui.display_timezone	enumeration	etc/utc	the timezone used to format timestamps in the ui [etc/utc = 0, america/new_york = 1]	application
-version	version	1000024.2-upgrading-to-1000024.3-step-022	set the active cluster version in the format '<major>.<minor>'	application
+version	version	1000024.3-upgrading-to-1000025.1-step-002	set the active cluster version in the format '<major>.<minor>'	application

docs/generated/settings/settings.html

@@ -360,6 +360,6 @@
 <tr><td><div id="setting-trace-zipkin-collector" class="anchored"><code>trace.zipkin.collector</code></div></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as &lt;host&gt;:&lt;port&gt;. If no port is specified, 9411 will be used.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-database-locality-metadata-enabled" class="anchored"><code>ui.database_locality_metadata.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-display-timezone" class="anchored"><code>ui.display_timezone</code></div></td><td>enumeration</td><td><code>etc/utc</code></td><td>the timezone used to format timestamps in the ui [etc/utc = 0, america/new_york = 1]</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
-<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000024.2-upgrading-to-1000024.3-step-022</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000024.3-upgrading-to-1000025.1-step-002</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 </tbody>
 </table>

pkg/build/version.txt

@@ -1 +1 @@
-v24.3.0-alpha.3
+v25.1.0-alpha.00000000

pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant

@@ -413,7 +413,7 @@ select crdb_internal.get_vmodule()
 query T
 SELECT crdb_internal.release_series(crdb_internal.node_executable_version())
 ----
-24.3
+25.1
 
 query ITTT colnames,rowsort
 select node_id, component, field, regexp_replace(regexp_replace(value, '^\d+$', '<port>'), e':\\d+', ':<port>') as value from crdb_internal.node_runtime_info
@@ -501,7 +501,7 @@ select * from crdb_internal.gossip_alerts
 query T
 SELECT crdb_internal.release_series(crdb_internal.node_executable_version())
 ----
-24.3
+25.1
 
 user root
 

pkg/cli/testdata/declarative-rules/invalid_version

@@ -4,6 +4,6 @@ invalid_version
 debug declarative-print-rules 1.1 op
 unsupported version number, the supported versions are: 
  latest
- latest
+ 1000024.3
  1000024.2
  1000024.1

pkg/clusterversion/README.md

@@ -183,7 +183,10 @@ avoid large changes on `master` which might cause merge conflicts for backports.
 
 - [ ] Update `PreviousRelease` constant
 
-- [ ] Add start version (e.g. `V24.2Start` with version `24.1-2`)
+- [ ] Add start version (e.g. `V24.2Start` with version `24.1-2`) and add a new
+  first upgrade for it (in `upgrades/upgrades.go`).
+
+- [ ] Update `roachpb.successorSeries` map and update `TestReleaseSeriesSuccessor`
 
 - [ ] Update `pkg/build/version.txt` to the new version (e.g. `v24.2.0-alpha.00000000`)
 
@@ -208,13 +211,11 @@ avoid large changes on `master` which might cause merge conflicts for backports.
   no longer have that code around. The data is obtained from the release branch
   (e.g. `release-24.1`) using the `sql-bootstrap-data` utility:
   ```
-  ./dev build sql-bootstrap-data
-  ./bin/sql-bootstrap-data
+  ./dev build sql-bootstrap-data && bin/sql-bootstrap-data
   ```
-  This will create a pair of files that need to be copied to
+  This will create a set of files that need to be copied to
   `pkg/sql/catalog/bootstrap/data` on the `master` branch; it will also output
-  what code modifications need to be performed. Note that the BUILD.bazel will
-- also need to be modified to include the new embedded files under `embedsrcs`.
+  what code modifications need to be performed.
 
 - [ ] Update releases file:
   ```

pkg/clusterversion/cockroach_versions.go

@@ -255,6 +255,11 @@ const (
 	// to the system.table_metadata table
 	V24_3_AddTableMetadataCols
 
+	// V24_3 is CockroachDB v24.3. It's used for all v24.3.x patch releases.
+	V24_3
+
+	V25_1_Start
+
 	// *************************************************
 	// Step (1) Add new versions above this comment.
 	// Do not add new versions to a patch release.
@@ -314,6 +319,11 @@ var versionTable = [numKeys]roachpb.Version{
 	V24_3_UseRACV2Full:                                 {Major: 24, Minor: 2, Internal: 20},
 	V24_3_AddTableMetadataCols:                         {Major: 24, Minor: 2, Internal: 22},
 
+	V24_3: {Major: 24, Minor: 3, Internal: 0},
+
+	// v25.1 versions. Internal versions must be even.
+	V25_1_Start: {Major: 24, Minor: 3, Internal: 2},
+
 	// *************************************************
 	// Step (2): Add new versions above this comment.
 	// Do not add new versions to a patch release.
@@ -327,19 +337,17 @@ const Latest Key = numKeys - 1
 // MinSupported is the minimum logical cluster version supported by this branch.
 const MinSupported Key = V24_2
 
-// PreviousRelease is the logical cluster version of the previous release.
-//
-// Note: this is always the last element of SupportedPreviousReleases(); it is
-// also provided as a constant for convenience.
+// PreviousRelease is the logical cluster version of the previous release (which must
+// have at least an RC build published).
 const PreviousRelease Key = V24_2
 
-// V24_3 is a placeholder that will eventually be replaced by the actual 24.3
+// V25_1 is a placeholder that will eventually be replaced by the actual 25.1
 // version Key, but in the meantime it points to the latest Key. The placeholder
 // is defined so that it can be referenced in code that simply wants to check if
 // a cluster is running 24.3 and has completed all associated migrations; most
 // version gates can use this instead of defining their own version key if they
-// only need to check that the cluster has upgraded to 24.3.
-const V24_3 = Latest
+// only need to check that the cluster has upgraded to 25.1.
+const V25_1 = Latest
 
 // DevelopmentBranch must be true on the main development branch but should be
 // set to false on a release branch once the set of versions becomes append-only
@@ -398,7 +406,7 @@ func (k Key) String() string {
 // cluster).
 func SupportedPreviousReleases() []Key {
 	res := make([]Key, 0, 2)
-	for k := MinSupported; k < Latest; k++ {
+	for k := MinSupported; k <= PreviousRelease; k++ {
 		if k.IsFinal() {
 			res = append(res, k)
 		}

pkg/cmd/release/update_releases.go

@@ -126,10 +126,10 @@ func processReleaseData(data []Release) map[string]release.Series {
 			continue
 		}
 
-		// For the purposes of the cockroach_releases file, we are only
-		// interested in beta and rc pre-releases, as we do not support
-		// upgrades from alpha releases.
-		if pre := v.PreRelease(); pre != "" && !strings.HasPrefix(pre, "rc") && !strings.HasPrefix(pre, "beta") {
+		// For the purposes of the cockroach_releases file, we are only interested
+		// in rc pre-releases, as we do not support upgrades from alpha or beta
+		// releases.
+		if pre := v.PreRelease(); pre != "" && !strings.HasPrefix(pre, "rc") {
 			continue
 		}
 		// Skip cloud-only releases, because the binaries are not yet publicly available.
@@ -322,7 +322,7 @@ func releaseName(name string) string {
 }
 
 func generateRepositoriesFile(versions ...string) error {
-	client := httputil.NewClientWithTimeout(15 * time.Second)
+	client := httputil.NewClientWithTimeout(45 * time.Second)
 	cfgKeys := map[string]string{
 		"CONFIG_LINUX_AMD64":  "linux-amd64",
 		"CONFIG_LINUX_ARM64":  "linux-arm64",
@@ -334,12 +334,13 @@ func generateRepositoriesFile(versions ...string) error {
 		versionToCfgToHash[v] = make(map[string]string)
 		for cfgKey, cfg := range cfgKeys {
 			url := fmt.Sprintf("https://binaries.cockroachdb.com/cockroach-v%s.%s.tgz", v, cfg)
+			fmt.Printf("getting %s\n", url)
 			resp, err := client.Get(context.Background(), url)
 			if err != nil {
 				return fmt.Errorf("could not download cockroach release: %w", err)
 			}
 			if resp.StatusCode != http.StatusOK {
-				return fmt.Errorf("unexpected status code %d when downloading %s", resp.StatusCode, url)
+				return fmt.Errorf("unexpected status code %d when downloading %s; consider turning off NetSkope", resp.StatusCode, url)
 			}
 			var blob bytes.Buffer
 			if _, err := io.Copy(&blob, resp.Body); err != nil {

pkg/cmd/release/update_releases_test.go

@@ -39,10 +39,6 @@ func Test_processReleaseData(t *testing.T) {
 			Predecessor: "22.2",
 			Withdrawn:   []string{"23.1.0"},
 		},
-		"23.2": {
-			Latest:      "23.2.0-beta.1",
-			Predecessor: "23.1",
-		},
 	}
 	require.Equal(t, expectedReleaseData, processReleaseData(data))
 }

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion_test.go

@@ -37,6 +37,7 @@ var testPredecessorMapping = map[string]*clusterupgrade.Version{
 	"24.1": clusterupgrade.MustParseVersion("v23.2.4"),
 	"24.2": clusterupgrade.MustParseVersion("v24.1.1"),
 	"24.3": clusterupgrade.MustParseVersion("v24.2.2"),
+	"25.1": clusterupgrade.MustParseVersion("v24.3.0"),
 }
 
 //go:embed testdata/test_releases.yaml
@@ -244,7 +245,7 @@ func Test_choosePreviousReleases(t *testing.T) {
 			arch:              vm.ArchAMD64,
 			numUpgrades:       3,
 			enableSkipVersion: true,
-			expectedReleases:  []string{"23.2.4", "24.1.1", "24.2.2"},
+			expectedReleases:  []string{"23.1.17", "23.2.4", "24.1.1"},
 		},
 	}
 

pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/skip_version_upgrades

@@ -28,72 +28,72 @@ ok
 plan
 ----
 Seed:               12345
-Upgrades:           v23.2.0 → v24.1.1 → v24.2.0 → <current>
+Upgrades:           v23.1.4 → v23.2.0 → v24.1.1 → <current>
 Deployment mode:    system-only
 Plan:
-├── install fixtures for version "v23.2.0" (1)
-├── start cluster at version "v23.2.0" (2)
-├── wait for all nodes (:1-4) to acknowledge cluster version '23.2' on system tenant (3)
+├── install fixtures for version "v23.1.4" (1)
+├── start cluster at version "v23.1.4" (2)
+├── wait for all nodes (:1-4) to acknowledge cluster version '23.1' on system tenant (3)
 ├── run "initialize bank workload" (4)
 ├── start background hooks concurrently
 │   ├── run "bank workload", after 3m0s delay (5)
 │   └── run "csv server", after 5s delay (6)
-├── upgrade cluster from "v23.2.0" to "v24.1.1"
+├── upgrade cluster from "v23.1.4" to "v23.2.0"
 │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (7)
-│   ├── upgrade nodes :1-4 from "v23.2.0" to "v24.1.1"
-│   │   ├── restart node 4 with binary version v24.1.1 (8)
-│   │   ├── restart node 1 with binary version v24.1.1 (9)
+│   ├── upgrade nodes :1-4 from "v23.1.4" to "v23.2.0"
+│   │   ├── restart node 4 with binary version v23.2.0 (8)
+│   │   ├── restart node 1 with binary version v23.2.0 (9)
 │   │   ├── run mixed-version hooks concurrently
 │   │   │   ├── run "mixed-version 1", after 500ms delay (10)
 │   │   │   └── run "mixed-version 2", after 5s delay (11)
-│   │   ├── restart node 2 with binary version v24.1.1 (12)
-│   │   └── restart node 3 with binary version v24.1.1 (13)
+│   │   ├── restart node 2 with binary version v23.2.0 (12)
+│   │   └── restart node 3 with binary version v23.2.0 (13)
 │   ├── allow upgrade to happen on system tenant by resetting `preserve_downgrade_option` (14)
 │   ├── run mixed-version hooks concurrently
 │   │   ├── run "mixed-version 1", after 100ms delay (15)
 │   │   └── run "mixed-version 2", after 500ms delay (16)
-│   ├── wait for all nodes (:1-4) to acknowledge cluster version '24.1' on system tenant (17)
+│   ├── wait for all nodes (:1-4) to acknowledge cluster version '23.2' on system tenant (17)
 │   └── run "validate upgrade" (18)
-├── upgrade cluster from "v24.1.1" to "v24.2.0"
+├── upgrade cluster from "v23.2.0" to "v24.1.1"
 │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (19)
-│   ├── upgrade nodes :1-4 from "v24.1.1" to "v24.2.0"
-│   │   ├── restart node 1 with binary version v24.2.0 (20)
-│   │   ├── restart node 3 with binary version v24.2.0 (21)
-│   │   ├── restart node 4 with binary version v24.2.0 (22)
+│   ├── upgrade nodes :1-4 from "v23.2.0" to "v24.1.1"
+│   │   ├── restart node 1 with binary version v24.1.1 (20)
+│   │   ├── restart node 3 with binary version v24.1.1 (21)
+│   │   ├── restart node 4 with binary version v24.1.1 (22)
 │   │   ├── run "mixed-version 1" (23)
-│   │   ├── restart node 2 with binary version v24.2.0 (24)
+│   │   ├── restart node 2 with binary version v24.1.1 (24)
 │   │   └── run "mixed-version 2" (25)
-│   ├── downgrade nodes :1-4 from "v24.2.0" to "v24.1.1"
-│   │   ├── restart node 4 with binary version v24.1.1 (26)
-│   │   ├── restart node 3 with binary version v24.1.1 (27)
-│   │   ├── restart node 2 with binary version v24.1.1 (28)
-│   │   └── restart node 1 with binary version v24.1.1 (29)
-│   ├── upgrade nodes :1-4 from "v24.1.1" to "v24.2.0"
-│   │   ├── restart node 3 with binary version v24.2.0 (30)
+│   ├── downgrade nodes :1-4 from "v24.1.1" to "v23.2.0"
+│   │   ├── restart node 4 with binary version v23.2.0 (26)
+│   │   ├── restart node 3 with binary version v23.2.0 (27)
+│   │   ├── restart node 2 with binary version v23.2.0 (28)
+│   │   └── restart node 1 with binary version v23.2.0 (29)
+│   ├── upgrade nodes :1-4 from "v23.2.0" to "v24.1.1"
+│   │   ├── restart node 3 with binary version v24.1.1 (30)
 │   │   ├── run "mixed-version 1" (31)
-│   │   ├── restart node 2 with binary version v24.2.0 (32)
-│   │   ├── restart node 1 with binary version v24.2.0 (33)
+│   │   ├── restart node 2 with binary version v24.1.1 (32)
+│   │   ├── restart node 1 with binary version v24.1.1 (33)
 │   │   ├── run "mixed-version 2" (34)
-│   │   └── restart node 4 with binary version v24.2.0 (35)
+│   │   └── restart node 4 with binary version v24.1.1 (35)
 │   ├── allow upgrade to happen on system tenant by resetting `preserve_downgrade_option` (36)
-│   ├── wait for all nodes (:1-4) to acknowledge cluster version '24.2' on system tenant (37)
+│   ├── wait for all nodes (:1-4) to acknowledge cluster version '24.1' on system tenant (37)
 │   └── run "validate upgrade" (38)
-└── upgrade cluster from "v24.2.0" to "<current>"
+└── upgrade cluster from "v24.1.1" to "<current>"
    ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (39)
-   ├── upgrade nodes :1-4 from "v24.2.0" to "<current>"
+   ├── upgrade nodes :1-4 from "v24.1.1" to "<current>"
    │   ├── restart node 3 with binary version <current> (40)
    │   ├── run "mixed-version 1" (41)
    │   ├── restart node 2 with binary version <current> (42)
    │   ├── run "mixed-version 2" (43)
    │   ├── restart node 4 with binary version <current> (44)
    │   └── restart node 1 with binary version <current> (45)
-   ├── downgrade nodes :1-4 from "<current>" to "v24.2.0"
-   │   ├── restart node 2 with binary version v24.2.0 (46)
-   │   ├── restart node 3 with binary version v24.2.0 (47)
-   │   ├── restart node 1 with binary version v24.2.0 (48)
+   ├── downgrade nodes :1-4 from "<current>" to "v24.1.1"
+   │   ├── restart node 2 with binary version v24.1.1 (46)
+   │   ├── restart node 3 with binary version v24.1.1 (47)
+   │   ├── restart node 1 with binary version v24.1.1 (48)
    │   ├── run "mixed-version 1" (49)
-   │   └── restart node 4 with binary version v24.2.0 (50)
-   ├── upgrade nodes :1-4 from "v24.2.0" to "<current>"
+   │   └── restart node 4 with binary version v24.1.1 (50)
+   ├── upgrade nodes :1-4 from "v24.1.1" to "<current>"
    │   ├── restart node 1 with binary version <current> (51)
    │   ├── restart node 3 with binary version <current> (52)
    │   ├── run "mixed-version 2" (53)

pkg/cmd/sql-bootstrap-data/main.go

@@ -32,7 +32,7 @@ func main() {
 	tenantKeys, tenantHash := bootstrap.GetAndHashInitialValuesToString(12345)
 
 	fmt.Printf("\n// Commit the generated files and " +
-		" add the following declarations to the end of initial_values.go:\n")
+		"add the following declarations to the end of initial_values.go:\n")
 	writeDataFile(version, "system.keys", systemKeys)
 	writeDataFile(version, "system.sha256", systemHash)
 	writeDataFile(version, "tenant.keys", tenantKeys)
@@ -45,7 +45,15 @@ func main() {
 		systemHash:    v%[1]d_%[2]d_system_sha256,
 		nonSystem:     v%[1]d_%[2]d_tenant_keys,
 		nonSystemHash: v%[1]d_%[2]d_tenant_sha256,
-	}.build
+	}.build,
+`, version.Major, version.Minor)
+
+	fmt.Printf("\n\n// Add these to BUILD.bazel, under embedsrcs:\n")
+	fmt.Printf(`
+        "data/%[1]d_%[2]d_system.keys",
+        "data/%[1]d_%[2]d_system.sha256",
+        "data/%[1]d_%[2]d_tenant.keys",
+        "data/%[1]d_%[2]d_tenant.sha256",
 `, version.Major, version.Minor)
 }
 

pkg/roachpb/version.go

@@ -235,6 +235,7 @@ var successorSeries = map[ReleaseSeries]ReleaseSeries{
 	{23, 2}: {24, 1},
 	{24, 1}: {24, 2},
 	{24, 2}: {24, 3},
+	{24, 3}: {25, 1},
 }
 
 // ReleaseSeries obtains the release series for the given version. Specifically:

pkg/roachpb/version_test.go

@@ -93,7 +93,7 @@ func TestReleaseSeriesSuccessor(t *testing.T) {
 	for ok := true; ok; r, ok = r.Successor() {
 		seq = append(seq, r.String())
 	}
-	expected := "20.1, 20.2, 21.1, 21.2, 22.1, 22.2, 23.1, 23.2, 24.1, 24.2, 24.3"
+	expected := "20.1, 20.2, 21.1, 21.2, 22.1, 22.2, 23.1, 23.2, 24.1, 24.2, 24.3, 25.1"
 	require.Equal(t, expected, strings.Join(seq, ", "))
 }
 

pkg/sql/catalog/bootstrap/BUILD.bazel

@@ -16,6 +16,10 @@ go_library(
         "data/24_2_system.sha256",
         "data/24_2_tenant.keys",
         "data/24_2_tenant.sha256",
+        "data/24_3_system.keys",
+        "data/24_3_system.sha256",
+        "data/24_3_tenant.keys",
+        "data/24_3_tenant.sha256",
     ],
     importpath = "github.com/cockroachdb/cockroach/pkg/sql/catalog/bootstrap",
     visibility = ["//visibility:public"],