Merge branch 'version/0-41-0-RC1' into DX-1754

activestate.generators.yaml

@@ -53,6 +53,15 @@ scripts:
     language: bash
     description: Detects new localisation calls and generates placeholder entries in en-us.yaml
     value: python3 scripts/locale-generator.py
+  - name: generate-payload
+    language: bash
+    description: Generate payload for installer / update archives
+    value: |
+      set -e
+      $constants.SET_ENV
+      
+      echo "# Generate payload"
+      go run ./scripts/ci/payload-generator/main.go
   - name: generate-update
     language: bash
     description: Generate update files
@@ -61,26 +70,13 @@ scripts:
       export GOARCH=${1:-amd64}
       $constants.SET_ENV
 
-      echo "# Create temp dir to generate bits"
-      TEMPDIR=$BUILD_TARGET_DIR/state-install
-      mkdir -p $TEMPDIR
-      cp -a $BUILD_TARGET_DIR/$constants.BUILD_INSTALLER_TARGET $TEMPDIR
-
-      echo "# Copy targets to temp dir"
-      BINDIR=$TEMPDIR/bin
-      mkdir -p $BINDIR
-      cp -a $BUILD_TARGET_DIR/$constants.BUILD_DAEMON_TARGET $BINDIR
-      cp -a $BUILD_TARGET_DIR/$constants.BUILD_EXEC_TARGET $BINDIR
-      cp -a $BUILD_TARGET_DIR/$constants.BUILD_TARGET $BINDIR
+      $scripts.generate-payload
 
       echo "# Create update dir"
       mkdir -p ./build/update
 
       echo "# Generate update from temp dir"
-      go run scripts/ci/update-generator/main.go -o ./build/update $TEMPDIR
-
-      echo "# Remove temp dir"
-      rm -rf $TEMPDIR
+      go run scripts/ci/update-generator/main.go -o ./build/update $PAYLOADDIR
   - name: generate-remote-install-deployment
     language: bash
     value: go run scripts/ci/deploy-generator/remote-installer/main.go "$@"

changelog.md

@@ -6,6 +6,47 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres
 to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### 0.40.0
+
+### Added
+
+- New command `state projects edit` which allows you to edit a projects name,
+  visibility, and linked git repository. You will need to opt-in to unstable
+  commands to use it.
+- New command `state projects delete` which allows you to delete a project.
+  You will need to opt-in to unstable commands to use it.
+- New command `state projects move` which allows you to move a project to a
+  different organization. You will need to opt-in to unstable commands to use
+  it.
+
+### Changed
+
+- Runtime installations have been updated to use our new buildplanner API. This
+  will enable us to develop new features in future versions. There should be no
+  impact to the user experience in this version.
+- Runtime installation is now atomic, meaning that an interruption to the
+  installation progress will not leave you in a corrupt state.
+- Requirement names are now normalized, avoiding requirement name collisions as
+  well as making it easier to install packages by their non-standard naming.
+- Commands which do not produce JSON output will now error out and say they do
+  not support JSON, rather than produce empty output.
+- When `state clean uninstall` cannot uninstall the State Tool because of third
+  party files in its installation dir it will now report what those files are.
+
+### Removed
+
+- The output format `--output=editor.v0` has been removed. Instead
+  use `--output=editor` or `--output=json`.
+
+### Fixed
+
+- Fixed issue where the `--namespace` flag on `state packages` was not
+  respected.
+- Fixed issue where `PYTHONTPATH` would not be set to empty when sourcing a
+  runtime, making it so that a system runtime can contaminate the sourced
+  runtime.
+- Several localization improvements.
+
 ### 0.39.0
 
 ### Added

cmd/state-installer/cmd.go

@@ -236,6 +236,26 @@ func execute(out output.Outputer, cfg *config.Instance, an analytics.Dispatcher,
 		params.path = installPath
 	}
 
+	// Detect if target dir is existing install of same target branch
+	var installedBranch string
+	marker := filepath.Join(installPath, installation.InstallDirMarker)
+	if stateToolInstalled && fileutils.TargetExists(marker) {
+		markerContents, err := fileutils.ReadFile(marker)
+		if err != nil {
+			return errs.Wrap(err, "Could not read marker file")
+		}
+		// The marker file is empty for versions prior to v0.40.0-RC3
+		if len(markerContents) > 0 {
+			var markerMeta installation.InstallMarkerMeta
+			if err := json.Unmarshal(markerContents, &markerMeta); err != nil {
+				return errs.Wrap(err, "Could not parse install marker file")
+			}
+			installedBranch = markerMeta.Branch
+		}
+	}
+	// Older state tools did not bake in meta information, in this case we allow overwriting regardless of branch
+	targetingSameBranch := installedBranch == "" || installedBranch == constants.BranchName
+
 	// If this is a fresh installation we ensure that the target directory is empty
 	if !stateToolInstalled && fileutils.DirExists(params.path) && !params.force {
 		empty, err := fileutils.IsEmptyDir(params.path)
@@ -257,7 +277,7 @@ func execute(out output.Outputer, cfg *config.Instance, an analytics.Dispatcher,
 	an.Event(AnalyticsFunnelCat, route)
 
 	// Check if state tool already installed
-	if !params.isUpdate && !params.force && stateToolInstalled {
+	if !params.isUpdate && !params.force && stateToolInstalled && !targetingSameBranch {
 		logging.Debug("Cancelling out because State Tool is already installed")
 		out.Print(fmt.Sprintf("State Tool Package Manager is already installed at [NOTICE]%s[/RESET]. To reinstall use the [ACTIONABLE]--force[/RESET] flag.", installPath))
 		an.Event(AnalyticsFunnelCat, "already-installed")

cmd/state-installer/test/integration/installer_int_test.go

@@ -36,19 +36,16 @@ func (suite *InstallerIntegrationTestSuite) TestInstallFromLocalSource() {
 	ts := e2e.New(suite.T(), false)
 	defer ts.Close()
 
-	suite.setupTest(ts)
 	suite.SetupRCFile(ts)
-	suite.T().Setenv("ACTIVESTATE_HOME", ts.Dirs.HomeDir)
-
-	target := filepath.Join(ts.Dirs.Work, "installation")
+	suite.T().Setenv(constants.HomeEnvVarName, ts.Dirs.HomeDir)
 
 	dir, err := ioutil.TempDir("", "system*")
 	suite.NoError(err)
 
 	// Run installer with source-path flag (ie. install from this local path)
 	cp := ts.SpawnCmdWithOpts(
 		suite.installerExe,
-		e2e.WithArgs(target),
+		e2e.WithArgs(installationDir(ts)),
 		e2e.AppendEnv(constants.DisableUpdates+"=false"),
 		e2e.AppendEnv(fmt.Sprintf("%s=%s", constants.OverwriteDefaultSystemPathEnvVarName, dir)),
 	)
@@ -68,22 +65,36 @@ func (suite *InstallerIntegrationTestSuite) TestInstallFromLocalSource() {
 	// Ensure installing overtop doesn't result in errors
 	cp = ts.SpawnCmdWithOpts(
 		suite.installerExe,
-		e2e.WithArgs(target, "--force"),
+		e2e.WithArgs(installationDir(ts)),
 		e2e.AppendEnv(constants.DisableUpdates+"=false"),
 		e2e.AppendEnv(fmt.Sprintf("%s=%s", constants.OverwriteDefaultSystemPathEnvVarName, dir)),
 	)
+	cp.Expect("successfully installed")
+	cp.WaitForInput()
+	cp.SendLine("exit")
+	cp.ExpectExitCode(0)
 
-	// Assert output
+	// Again ensure installing overtop doesn't result in errors, but mock an older state tool format where
+	// the marker has no contents
+	suite.Require().NoError(fileutils.WriteFile(filepath.Join(installationDir(ts), installation.InstallDirMarker), []byte{}))
+	cp = ts.SpawnCmdWithOpts(
+		suite.installerExe,
+		e2e.WithArgs(installationDir(ts)),
+		e2e.AppendEnv(constants.DisableUpdates+"=false"),
+		e2e.AppendEnv(fmt.Sprintf("%s=%s", constants.OverwriteDefaultSystemPathEnvVarName, dir)),
+	)
 	cp.Expect("successfully installed")
 
-	stateExec, err := installation.StateExecFromDir(target)
-	suite.Contains(stateExec, target, "Ensure we're not grabbing state tool from integration test bin dir")
+	installDir := installationDir(ts)
+
+	stateExec, err := installation.StateExecFromDir(installDir)
+	suite.Contains(stateExec, installDir, "Ensure we're not grabbing state tool from integration test bin dir")
 	suite.NoError(err)
 
 	stateExecResolved, err := fileutils.ResolvePath(stateExec)
 	suite.Require().NoError(err)
 
-	serviceExec, err := installation.ServiceExecFromDir(target)
+	serviceExec, err := installation.ServiceExecFromDir(installDir)
 	suite.NoError(err)
 
 	// Verify that launched subshell has State tool on PATH
@@ -126,14 +137,10 @@ func (suite *InstallerIntegrationTestSuite) TestInstallIncompatible() {
 	ts := e2e.New(suite.T(), false)
 	defer ts.Close()
 
-	suite.setupTest(ts)
-
-	target := filepath.Join(ts.Dirs.Work, "installation")
-
 	// Run installer with source-path flag (ie. install from this local path)
 	cp := ts.SpawnCmdWithOpts(
 		suite.installerExe,
-		e2e.WithArgs(target),
+		e2e.WithArgs(installationDir(ts)),
 		e2e.AppendEnv(constants.DisableUpdates+"=false", sysinfo.VersionOverrideEnvVar+"=10.0.0"),
 	)
 
@@ -147,16 +154,12 @@ func (suite *InstallerIntegrationTestSuite) TestInstallNoErrorTips() {
 	ts := e2e.New(suite.T(), false)
 	defer ts.Close()
 
-	suite.setupTest(ts)
-
-	target := filepath.Join(ts.Dirs.Work, "installation")
-
 	dir, err := ioutil.TempDir("", "system*")
 	suite.NoError(err)
 
 	cp := ts.SpawnCmdWithOpts(
 		suite.installerExe,
-		e2e.WithArgs(target, "--activate", "ActiveState/DoesNotExist"),
+		e2e.WithArgs(installationDir(ts), "--activate", "ActiveState/DoesNotExist"),
 		e2e.AppendEnv(constants.DisableUpdates+"=true"),
 		e2e.AppendEnv(fmt.Sprintf("%s=%s", constants.OverwriteDefaultSystemPathEnvVarName, dir)),
 	)
@@ -170,16 +173,12 @@ func (suite *InstallerIntegrationTestSuite) TestInstallErrorTips() {
 	ts := e2e.New(suite.T(), false)
 	defer ts.Close()
 
-	suite.setupTest(ts)
-
-	target := filepath.Join(ts.Dirs.Work, "installation")
-
 	dir, err := ioutil.TempDir("", "system*")
 	suite.NoError(err)
 
 	cp := ts.SpawnCmdWithOpts(
 		suite.installerExe,
-		e2e.WithArgs(target, "--activate", "ActiveState-CLI/Python3"),
+		e2e.WithArgs(installationDir(ts), "--activate", "ActiveState-CLI/Python3"),
 		e2e.AppendEnv(constants.DisableUpdates+"=true"),
 		e2e.AppendEnv(fmt.Sprintf("%s=%s", constants.OverwriteDefaultSystemPathEnvVarName, dir)),
 	)
@@ -197,9 +196,7 @@ func (suite *InstallerIntegrationTestSuite) TestStateTrayRemoval() {
 	ts := e2e.New(suite.T(), false)
 	defer ts.Close()
 
-	suite.setupTest(ts)
-
-	dir := filepath.Join(ts.Dirs.Work, "installation")
+	dir := installationDir(ts)
 
 	// Install a release version that still has state-tray.
 	version := "0.35.0-SHAb78e2a4"
@@ -269,7 +266,7 @@ func (suite *InstallerIntegrationTestSuite) TestInstallerOverwriteServiceApp() {
 
 	cp := ts.SpawnCmdWithOpts(
 		suite.installerExe,
-		e2e.WithArgs(filepath.Join(ts.Dirs.Work, "installation")),
+		e2e.WithArgs(installationDir(ts)),
 		e2e.AppendEnv(fmt.Sprintf("%s=%s", constants.AppInstallDirOverrideEnvVarName, appInstallDir)),
 	)
 	cp.Expect("Done")
@@ -279,7 +276,7 @@ func (suite *InstallerIntegrationTestSuite) TestInstallerOverwriteServiceApp() {
 	// State Service.app should be overwritten cleanly without error.
 	cp = ts.SpawnCmdWithOpts(
 		suite.installerExe,
-		e2e.WithArgs(filepath.Join(ts.Dirs.Work, "installation2")),
+		e2e.WithArgs(installationDir(ts)+"2"),
 		e2e.AppendEnv(fmt.Sprintf("%s=%s", constants.AppInstallDirOverrideEnvVarName, appInstallDir)),
 	)
 	cp.Expect("Done")
@@ -333,18 +330,18 @@ func (suite *InstallerIntegrationTestSuite) AssertConfig(ts *e2e.Session) {
 	}
 }
 
-func (s *InstallerIntegrationTestSuite) setupTest(ts *e2e.Session) {
-	root := environment.GetRootPathUnsafe()
-	buildDir := fileutils.Join(root, "build")
-	installerExe := filepath.Join(buildDir, constants.StateInstallerCmd+osutils.ExeExt)
-	if !fileutils.FileExists(installerExe) {
-		s.T().Fatal("E2E tests require a state-installer binary. Run `state run build-installer`.")
-	}
-	s.installerExe = ts.CopyExeToDir(installerExe, filepath.Join(ts.Dirs.Base, "installer"))
+func installationDir(ts *e2e.Session) string {
+	return filepath.Join(ts.Dirs.Work, "installation")
+}
+
+func (suite *InstallerIntegrationTestSuite) SetupSuite() {
+	localPayload := filepath.Join(environment.GetRootPathUnsafe(), "build", "payload")
+	suite.Assert().DirExists(localPayload, "locally generated payload exists")
+
+	installerExe := filepath.Join(localPayload, constants.StateInstallerCmd+osutils.ExeExt)
+	suite.Assert().FileExists(installerExe, "locally generated installer exists")
 
-	payloadDir := filepath.Dir(s.installerExe)
-	ts.CopyExeToDir(ts.Exe, filepath.Join(payloadDir, installation.BinDirName))
-	ts.CopyExeToDir(ts.SvcExe, filepath.Join(payloadDir, installation.BinDirName))
+	suite.installerExe = installerExe
 }
 
 func TestInstallerIntegrationTestSuite(t *testing.T) {

cmd/state/internal/cmdtree/checkout.go

@@ -29,6 +29,11 @@ func newCheckoutCommand(prime *primer.Values) *captain.Command {
 				Description: locale.Tl("flag_state_checkout_runtime-path_description", "Path to store the runtime files"),
 				Value:       &params.RuntimePath,
 			},
+			{
+				Name:        "no-clone",
+				Description: locale.Tl("flag_state_checkout_no_clone_description", "Do not clone the github repository associated with this project (if any)"),
+				Value:       &params.NoClone,
+			},
 		},
 		[]*captain.Argument{
 			{

cmd/state/internal/cmdtree/cmdtree.go

@@ -158,6 +158,7 @@ func New(prime *primer.Values, args ...string) *CmdTree {
 
 	stateCmd := newStateCommand(globals, prime)
 	stateCmd.AddChildren(
+		newHelloCommand(prime),
 		newActivateCommand(prime),
 		newInitCommand(prime),
 		newPushCommand(prime),
@@ -206,11 +207,6 @@ func New(prime *primer.Values, args ...string) *CmdTree {
 		newPublish(prime),
 	)
 
-	if !condition.OnCI() {
-		helloCmd := newHelloCommand(prime)
-		stateCmd.AddChildren(helloCmd)
-	}
-
 	return &CmdTree{
 		cmd: stateCmd,
 	}

cmd/state/internal/cmdtree/hello_example.go

@@ -14,7 +14,7 @@ func newHelloCommand(prime *primer.Values) *captain.Command {
 
 	cmd := captain.NewCommand(
 		// The command's name should not be localized as we want commands to behave consistently regardless of localization.
-		"hello",
+		"_hello",
 		// The title is printed with title formatting when running the command. Leave empty to disable.
 		locale.Tl("hello_cmd_title", "Saying hello"),
 		// The description is shown on --help output
@@ -57,9 +57,9 @@ func newHelloCommand(prime *primer.Values) *captain.Command {
 	// The group is used to group together commands in the --help output
 	cmd.SetGroup(UtilsGroup)
 	// Any new command should be marked unstable for the first release it goes out in.
-	// cmd.SetUnstable(true)
+	cmd.SetUnstable(true)
 	// Certain commands like `state deploy` are there for backwards compatibility, but we don't want to show them in the --help output as they are not part of the happy path or our long term goals.
-	// cmd.SetHidden(true)
+	cmd.SetHidden(true)
 
 	return cmd
 }

cmd/state/main.go

@@ -198,7 +198,7 @@ func run(args []string, isInteractive bool, cfg *config.Instance, out output.Out
 	defer events.Close("auth", auth.Close)
 
 	if err := auth.Sync(); err != nil {
-		logging.Warning("Could not sync authenticated state: %s", err.Error())
+		logging.Warning("Could not sync authenticated state: %s", errs.JoinMessage(err))
 	}
 
 	an := anAsync.New(svcmodel, cfg, auth, out, pjNamespace)

internal/access/access.go

@@ -10,28 +10,12 @@ import (
 // Secrets determines whether the authorized user has access
 // to the current project's secrets
 func Secrets(orgName string, auth *authentication.Auth) (bool, error) {
-	if isProjectOwner(orgName, auth) {
-		return true, nil
-	}
-
-	return isOrgMember(orgName, auth)
-}
-
-func isProjectOwner(orgName string, auth *authentication.Auth) bool {
-	if orgName != auth.WhoAmI() {
-		return false
-	}
-	return true
-}
-
-func isOrgMember(orgName string, auth *authentication.Auth) (bool, error) {
 	_, err := model.FetchOrgMember(orgName, auth.WhoAmI(), auth)
 	if err != nil {
 		if errors.Is(err, model.ErrMemberNotFound) {
 			return false, nil
 		}
 		return false, err
 	}
-
 	return true, nil
 }