From a1dd629263ff147ad2941e75a89550bf22c8bcbd Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 12 Oct 2023 13:28:43 -0400 Subject: [PATCH 1/6] Initial support for a master state tool version list. --- .github/workflows/build.yml | 5 ++ activestate.generators.yaml | 21 +++++- scripts/ci/update-version-list/main.go | 97 ++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 scripts/ci/update-version-list/main.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e42f94901b..5ad5397112 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -465,6 +465,11 @@ jobs: aws-region: ${{ env.AWS_REGION }} mask-aws-account-id: true + - # === Generate updated master versions.json if necessary === + name: Generate version list + shell: bash + run: state run generate-versions-list + - # === Deploy === name: Deploy shell: bash diff --git a/activestate.generators.yaml b/activestate.generators.yaml index f7e639c843..ea9925fc5d 100644 --- a/activestate.generators.yaml +++ b/activestate.generators.yaml @@ -59,7 +59,7 @@ scripts: value: | set -e $constants.SET_ENV - + echo "# Generate payload" go run ./scripts/ci/payload-generator/main.go "$@" - name: generate-update @@ -120,3 +120,22 @@ scripts: go run scripts/ci/payload-generator/main.go -b ${TEST_CHANNEL} -v ${TEST_VERSION} copy_test_payload go run scripts/ci/update-generator/main.go -b ${TEST_CHANNEL} -v ${TEST_VERSION} -o ${TEST_ARCHIVE_DIR} + - name: generate-versions-list + language: bash + standalone: true + description: Generates master versions.json from S3 and info.json's from generate-update + value: | + set -e + $constants.SET_ENV + + go run scripts/ci/update-version-list/main.go ./build/update + - name: generate-test-versions-list + language: bash + description: Generates test versions.json + value: | + set -e + $constants.SET_ENV + + $scripts.generate-test-update + + go run scripts/ci/update-version-list/main.go ./build/test-update \ No newline at end of file diff --git a/scripts/ci/update-version-list/main.go b/scripts/ci/update-version-list/main.go new file mode 100644 index 0000000000..6ee19b4ac7 --- /dev/null +++ b/scripts/ci/update-version-list/main.go @@ -0,0 +1,97 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/thoas/go-funk" + + "github.com/ActiveState/cli/internal/condition" + "github.com/ActiveState/cli/internal/fileutils" + "github.com/ActiveState/cli/internal/httputil" +) + +// Where the master version file lives on S3. +const S3PrefixURL = "https://state-tool.s3.amazonaws.com/" +const S3Bucket = "update/state/" +const VersionsJson = "versions.json" + +// Valid channels to update the master version file with. +var ValidChannels = []string{"master", "beta", "release", "LTS"} + +func init() { + if !condition.OnCI() { + // Allow testing with artifacts produced by `state run generate-test-update` + ValidChannels = append(ValidChannels, "test-channel") + } +} + +func main() { + if len(os.Args) != 2 { + log.Fatalf("Usage: %s ", os.Args[0]) + } + + // Fetch the current master list from S3. + versions := []map[string]string{} + fmt.Printf("Fetching master %s file from S3\n", VersionsJson) + bytes, err := httputil.Get(S3PrefixURL + S3Bucket + VersionsJson) + if err != nil { + log.Fatalf("Failed to fetch file: %s", err.Error()) + } + err = json.Unmarshal(bytes, &versions) + if err != nil { + log.Fatalf("Failed to decode JSON: %s", err.Error()) + } + + // Find info.json files to add to the master list and add them. + updated := false + buildDir := os.Args[1] + fmt.Printf("Searching for info.json files in %s\n", buildDir) + files, err := fileutils.ListDir(buildDir, false) + if err != nil { + log.Fatalf("Failed to search %s: %s", buildDir, err.Error()) + } + for _, file := range files { + if file.Name() != "info.json" { + continue + } + channel := strings.Split(file.RelativePath(), string(filepath.Separator))[0] + if !funk.Contains(ValidChannels, channel) { + continue + } + fmt.Printf("Found %s\n", file.RelativePath()) + bytes, err := fileutils.ReadFile(file.Path()) + if err != nil { + log.Fatalf("Unable to read file: %s", err.Error()) + } + info := map[string]string{} + err = json.Unmarshal(bytes, &info) + if err != nil { + log.Fatalf("Unable to decode JSON: %s", err.Error()) + } + info["path"] = S3PrefixURL + S3Bucket + info["path"] // convert relative path to full URL + versions = append(versions, info) + updated = true + } + + if !updated { + fmt.Println("No updates found.") + return + } + + // Write the updated list to disk. The s3-deployer script should pick it up and upload it. + localVersionsJson := filepath.Join(buildDir, VersionsJson) + fmt.Printf("Writing updated %s locally to %s\n", VersionsJson, localVersionsJson) + bytes, err = json.Marshal(versions) + if err != nil { + log.Fatalf("Failed to encode JSON: %s", err.Error()) + } + err = fileutils.WriteFile(localVersionsJson, bytes) + if err != nil { + log.Fatalf("Failed to write file: %s", err.Error()) + } +} From 0e255c2d608e8e9cf63dff4586c849da8a3ebc86 Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 12 Oct 2023 13:29:25 -0400 Subject: [PATCH 2/6] Temporarily allow PR channel for testing. --- .github/workflows/build.yml | 1 - scripts/ci/update-version-list/main.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5ad5397112..779b116fa7 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -401,7 +401,6 @@ jobs: SHELL: bash GITHUB_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} timeout-minutes: 10 - if: contains(fromJSON('["refs/heads/master", "refs/heads/beta", "refs/heads/release", "refs/heads/LTS"]'), github.ref) || startsWith(github.event.pull_request.head.ref, 'version/') # === Deploy Steps === steps: diff --git a/scripts/ci/update-version-list/main.go b/scripts/ci/update-version-list/main.go index 6ee19b4ac7..744e0af09f 100644 --- a/scripts/ci/update-version-list/main.go +++ b/scripts/ci/update-version-list/main.go @@ -21,7 +21,7 @@ const S3Bucket = "update/state/" const VersionsJson = "versions.json" // Valid channels to update the master version file with. -var ValidChannels = []string{"master", "beta", "release", "LTS"} +var ValidChannels = []string{"master", "beta", "release", "LTS", "mitchell"} func init() { if !condition.OnCI() { From a3f0c460ce63732a2826e154d75146617474e6dc Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 12 Oct 2023 14:06:31 -0400 Subject: [PATCH 3/6] Install state tool for deploy job. This was missed in #2784. --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 779b116fa7..45d3ca3126 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -415,6 +415,10 @@ jobs: with: go-version: ${{ matrix.go-version }} + - # === Install State Tool === + name: Install State Tool + uses: ActiveState/setup-state-tool@v1 + - # === Setup === name: Setup shell: bash From 711ff43e12772dec2a7dbac8858a155bfacb4b0c Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 12 Oct 2023 14:22:03 -0400 Subject: [PATCH 4/6] Revert "Temporarily allow PR channel for testing." This reverts commit 0e255c2d608e8e9cf63dff4586c849da8a3ebc86. --- .github/workflows/build.yml | 1 + scripts/ci/update-version-list/main.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45d3ca3126..86dae8ce58 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -401,6 +401,7 @@ jobs: SHELL: bash GITHUB_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} timeout-minutes: 10 + if: contains(fromJSON('["refs/heads/master", "refs/heads/beta", "refs/heads/release", "refs/heads/LTS"]'), github.ref) || startsWith(github.event.pull_request.head.ref, 'version/') # === Deploy Steps === steps: diff --git a/scripts/ci/update-version-list/main.go b/scripts/ci/update-version-list/main.go index 744e0af09f..6ee19b4ac7 100644 --- a/scripts/ci/update-version-list/main.go +++ b/scripts/ci/update-version-list/main.go @@ -21,7 +21,7 @@ const S3Bucket = "update/state/" const VersionsJson = "versions.json" // Valid channels to update the master version file with. -var ValidChannels = []string{"master", "beta", "release", "LTS", "mitchell"} +var ValidChannels = []string{"master", "beta", "release", "LTS"} func init() { if !condition.OnCI() { From 21403df6651c73a4ae7838c9f9ffaa1e58e543af Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 12 Oct 2023 14:39:47 -0400 Subject: [PATCH 5/6] Do not pollute master version list with master channel. --- scripts/ci/update-version-list/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/update-version-list/main.go b/scripts/ci/update-version-list/main.go index 6ee19b4ac7..8ebbe2e3bc 100644 --- a/scripts/ci/update-version-list/main.go +++ b/scripts/ci/update-version-list/main.go @@ -21,7 +21,7 @@ const S3Bucket = "update/state/" const VersionsJson = "versions.json" // Valid channels to update the master version file with. -var ValidChannels = []string{"master", "beta", "release", "LTS"} +var ValidChannels = []string{"beta", "release", "LTS"} func init() { if !condition.OnCI() { From 1c465a648b90b4a0e16c2dea3f8d8fa89ec7d53c Mon Sep 17 00:00:00 2001 From: mitchell Date: Thu, 12 Oct 2023 16:04:13 -0400 Subject: [PATCH 6/6] Address PR feedback. --- activestate.generators.yaml | 10 ---------- scripts/ci/update-version-list/main.go | 10 ++++++---- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/activestate.generators.yaml b/activestate.generators.yaml index ea9925fc5d..4096dbbb11 100644 --- a/activestate.generators.yaml +++ b/activestate.generators.yaml @@ -129,13 +129,3 @@ scripts: $constants.SET_ENV go run scripts/ci/update-version-list/main.go ./build/update - - name: generate-test-versions-list - language: bash - description: Generates test versions.json - value: | - set -e - $constants.SET_ENV - - $scripts.generate-test-update - - go run scripts/ci/update-version-list/main.go ./build/test-update \ No newline at end of file diff --git a/scripts/ci/update-version-list/main.go b/scripts/ci/update-version-list/main.go index 8ebbe2e3bc..681ba4b108 100644 --- a/scripts/ci/update-version-list/main.go +++ b/scripts/ci/update-version-list/main.go @@ -11,8 +11,10 @@ import ( "github.com/thoas/go-funk" "github.com/ActiveState/cli/internal/condition" + "github.com/ActiveState/cli/internal/constants" "github.com/ActiveState/cli/internal/fileutils" "github.com/ActiveState/cli/internal/httputil" + "github.com/ActiveState/cli/internal/updater" ) // Where the master version file lives on S3. @@ -21,7 +23,7 @@ const S3Bucket = "update/state/" const VersionsJson = "versions.json" // Valid channels to update the master version file with. -var ValidChannels = []string{"beta", "release", "LTS"} +var ValidChannels = []string{constants.BetaBranch, constants.ReleaseBranch} func init() { if !condition.OnCI() { @@ -36,7 +38,7 @@ func main() { } // Fetch the current master list from S3. - versions := []map[string]string{} + versions := []updater.AvailableUpdate{} fmt.Printf("Fetching master %s file from S3\n", VersionsJson) bytes, err := httputil.Get(S3PrefixURL + S3Bucket + VersionsJson) if err != nil { @@ -68,12 +70,12 @@ func main() { if err != nil { log.Fatalf("Unable to read file: %s", err.Error()) } - info := map[string]string{} + info := updater.AvailableUpdate{} err = json.Unmarshal(bytes, &info) if err != nil { log.Fatalf("Unable to decode JSON: %s", err.Error()) } - info["path"] = S3PrefixURL + S3Bucket + info["path"] // convert relative path to full URL + info.Path = S3PrefixURL + S3Bucket + info.Path // convert relative path to full URL versions = append(versions, info) updated = true }