0.5.0 (#29)

0.5.0

.goreleaser.yml

@@ -41,4 +41,3 @@ release:
   # disable: true
 changelog:
   disable: true
-

Makefile

@@ -48,4 +48,13 @@ go.build:
 	chmod +x dist/$(BIN)_v$(VERSION)_$(OS_TYPE)_$(ARCH)
 
 build-linux:
-	GOOS=linux OS_TYPE=linux $(MAKE) go.build
+	GOOS=linux OS_TYPE=linux $(MAKE) go.build
+
+run-docker-compose:
+	cp dist/anklet_v$(VERSION)*_linux_$(ARCH).zip docker/
+	cd docker && \
+		rm -f anklet_linux_$(ARCH) && \
+		unzip anklet_v$(VERSION)*_linux_$(ARCH).zip && \
+		mv anklet_v$(VERSION)*_linux_$(ARCH) anklet_linux_$(ARCH) && \
+		rm -f anklet_v$(VERSION)*_linux_$(ARCH).zip && \
+		docker-compose up --build --force-recreate

README.md

@@ -104,7 +104,6 @@ With the github plugin, there is a Receiver Plugin and a Handler Plugin.
           private_key: /Users/nathanpierce/veertuinc-anklet.2024-07-19.private-key.pem
           app_id: 949431
           installation_id: 52970581
-          registration: repo
           repo: anklet
           owner: veertuinc
           registry_url: http://anka.registry:8089
@@ -119,7 +118,6 @@ With the github plugin, there is a Receiver Plugin and a Handler Plugin.
       - name: RUNNER2
           plugin: github
           token: github_pat_1XXXXX
-          registration: repo
           repo: anklet
           owner: veertuinc
           registry_url: http://anka.registry:8089
@@ -258,6 +256,8 @@ Metrics for monitoring are available at `http://127.0.0.1:8080/metrics?format=js
 
 #### Prometheus
 
+- If repo is not set, the metrics will not show `repo=`.
+
 ```
 total_running_vms 0
 total_successful_runs_since_start 2

VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

docker/docker-compose.yml

@@ -12,8 +12,19 @@ services:
       - "8081:8081"
     environment:
       # - LOG_LEVEL=dev
-      - ANKLET_CONFIG_FILE_NAME=receiver-config.yml
-      # AGGREGATOR ###################################################
+      - ANKLET_CONFIG_FILE_NAME=org-receiver-config.yml
+      ##################################################################################
+      # ANKLET_GLOBAL_PRIVATE_KEY allows you to use the same private key for all runners
+      # instead of defining a different one in the config.yml file for each runner.
+      # You can also set it to be the PEM contents, vs pointing to a file.
+      # - ANKLET_GLOBAL_PRIVATE_KEY=/Users/nathanpierce/veertuinc-anklet.2024-07-19.private-key.pem
+      # - |-
+      #   ANKLET_GLOBAL_PRIVATE_KEY=-----BEGIN RSA PRIVATE KEY-----
+      #   MIIEpAIBAAKCAQEAyqZDXOEzV5gRocWAhH73chGjn4HBh1UCG2Du4v9JmcjfyYT3
+      #   . . .
+      #   -----END RSA PRIVATE KEY-----
+      ################################################################
+      # AGGREGATOR
       # - ANKLET_METRICS_AGGREGATOR=true
       # - ANKLET_METRICS_PORT=8081 
       # - ANKLET_METRICS_URLS=http://host.docker.internal:8080/metrics # comma separation for multiple urls

internal/anka/cli.go

@@ -106,6 +106,7 @@ func (cli *Cli) Execute(pluginCtx context.Context, args ...string) ([]byte, int,
 		case <-ticker.C:
 			logger.InfoContext(pluginCtx, fmt.Sprintf("execution of command %v is still in progress...", args))
 		case err := <-done:
+			// logger.InfoContext(pluginCtx, fmt.Sprintf("execution of command %v completed", args))
 			exitCode := 0
 			if exitErr, ok := err.(*exec.ExitError); ok {
 				exitCode = exitErr.ExitCode()
@@ -208,6 +209,7 @@ func (cli *Cli) AnkaDelete(workerCtx context.Context, pluginCtx context.Context,
 	deleteOutput, err := cli.ExecuteParseJson(pluginCtx, "anka", "-j", "delete", "--yes", vm.Name)
 	if err != nil {
 		logger.ErrorContext(pluginCtx, "error executing anka delete", "err", err)
+		cli.Execute(pluginCtx, "anka", "delete", "--yes", vm.Name)
 		return err
 	}
 	logger.DebugContext(pluginCtx, "successfully deleted vm", "std", deleteOutput.Message)
@@ -396,7 +398,7 @@ func (cli *Cli) EnsureVMTemplateExists(workerCtx context.Context, pluginCtx cont
 		}
 		defer globals.PullLock.Unlock()
 		pullJson, err := cli.AnkaRegistryPull(workerCtx, pluginCtx, targetTemplate, targetTag)
-		if pullJson.Code == 3 { // registry doesn't have template (or tag)
+		if pullJson == nil || pullJson.Code == 3 { // registry doesn't have template (or tag)
 			return err, nil
 		}
 		if err != nil {

internal/config/config.go

@@ -15,12 +15,13 @@ import (
 type ContextKey string
 
 type Config struct {
-	Plugins    []Plugin `yaml:"plugins"`
-	Log        Log      `yaml:"log"`
-	PidFileDir string   `yaml:"pid_file_dir"`
-	LogFileDir string   `yaml:"log_file_dir"`
-	WorkDir    string   `yaml:"work_dir"`
-	Metrics    Metrics  `yaml:"metrics"`
+	Plugins          []Plugin `yaml:"plugins"`
+	Log              Log      `yaml:"log"`
+	PidFileDir       string   `yaml:"pid_file_dir"`
+	LogFileDir       string   `yaml:"log_file_dir"`
+	WorkDir          string   `yaml:"work_dir"`
+	Metrics          Metrics  `yaml:"metrics"`
+	GlobalPrivateKey string   `yaml:"global_private_key"`
 }
 
 type Log struct {
@@ -54,7 +55,6 @@ type Plugin struct {
 	Name           string   `yaml:"name"`
 	Plugin         string   `yaml:"plugin"`
 	Token          string   `yaml:"token"`
-	Registration   string   `yaml:"registration"`
 	Repo           string   `yaml:"repo"`
 	Owner          string   `yaml:"owner"`
 	Database       Database `yaml:"database"`
@@ -67,6 +67,7 @@ type Plugin struct {
 	Secret         string   `yaml:"secret"`
 	HookID         int64    `yaml:"hook_id"`
 	SkipRedeliver  bool     `yaml:"skip_redeliver"`
+	RunnerGroup    string   `yaml:"runner_group"`
 }
 
 func LoadConfig(configPath string) (Config, error) {
@@ -146,6 +147,10 @@ func LoadInEnvs(config Config) (Config, error) {
 	if workDir != "" {
 		config.WorkDir = workDir
 	}
+	envGlobalPrivateKey := os.Getenv("ANKLET_GLOBAL_PRIVATE_KEY")
+	if envGlobalPrivateKey != "" {
+		config.GlobalPrivateKey = envGlobalPrivateKey
+	}
 	// pidFileDir := os.Getenv("ANKLET_PID_FILE_DIR")
 	// if pidFileDir != "" {
 	// 	config.PidFileDir = pidFileDir
@@ -194,3 +199,19 @@ func GetLoadedConfigFromContext(ctx context.Context) *Config {
 	}
 	return config
 }
+
+func GetIsRepoSetFromContext(ctx context.Context) bool {
+	isRepoSet, ok := ctx.Value(ContextKey("isRepoSet")).(bool)
+	if !ok {
+		panic("GetIsRepoSetFromContext failed")
+	}
+	return isRepoSet
+}
+
+func GetConfigFileNameFromContext(ctx context.Context) string {
+	configFileName, ok := ctx.Value(ContextKey("configFileName")).(string)
+	if !ok {
+		panic("GetConfigFileNameFromContext failed")
+	}
+	return configFileName
+}

internal/metrics/aggregator.go

@@ -95,7 +95,9 @@ func (s *Server) handleAggregatorPrometheusMetrics(workerCtx context.Context, lo
 				pluginName = pluginMap["plugin_name"].(string)
 				Name = pluginMap["name"].(string)
 				ownerName = pluginMap["owner_name"].(string)
-				repoName = pluginMap["repo_name"].(string)
+				if pluginMap["repo_name"] != nil {
+					repoName = pluginMap["repo_name"].(string)
+				}
 				status = pluginMap["status"].(string)
 				statusSince, err = time.Parse(time.RFC3339, pluginMap["status_since"].(string))
 				if err != nil {
@@ -116,12 +118,25 @@ func (s *Server) handleAggregatorPrometheusMetrics(workerCtx context.Context, lo
 						return
 					}
 				}
-				w.Write([]byte(fmt.Sprintf("plugin_status{name=%s,owner=%s,repo=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, metricsURL, status)))
+				if repoName == "" {
+					w.Write([]byte(fmt.Sprintf("plugin_status{name=%s,owner=%s,metricsUrl=%s} %s\n", Name, ownerName, metricsURL, status)))
+				} else {
+					w.Write([]byte(fmt.Sprintf("plugin_status{name=%s,owner=%s,repo=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, metricsURL, status)))
+				}
 				if !strings.Contains(pluginName, "_receiver") {
-					w.Write([]byte(fmt.Sprintf("plugin_last_successful_run{name=%s,owner=%s,repo=%s,job_url=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, lastSuccessfulRunJobUrl, metricsURL, lastSuccessfulRun.Format(time.RFC3339))))
-					w.Write([]byte(fmt.Sprintf("plugin_last_failed_run{name=%s,owner=%s,repo=%s,job_url=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, lastFailedRunJobUrl, metricsURL, lastFailedRun.Format(time.RFC3339))))
+					if repoName == "" {
+						w.Write([]byte(fmt.Sprintf("plugin_last_successful_run{name=%s,owner=%s,job_url=%s,metricsUrl=%s} %s\n", Name, ownerName, lastSuccessfulRunJobUrl, metricsURL, lastSuccessfulRun.Format(time.RFC3339))))
+						w.Write([]byte(fmt.Sprintf("plugin_last_failed_run{name=%s,owner=%s,job_url=%s,metricsUrl=%s} %s\n", Name, ownerName, lastFailedRunJobUrl, metricsURL, lastFailedRun.Format(time.RFC3339))))
+					} else {
+						w.Write([]byte(fmt.Sprintf("plugin_last_successful_run{name=%s,owner=%s,repo=%s,job_url=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, lastSuccessfulRunJobUrl, metricsURL, lastSuccessfulRun.Format(time.RFC3339))))
+						w.Write([]byte(fmt.Sprintf("plugin_last_failed_run{name=%s,owner=%s,repo=%s,job_url=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, lastFailedRunJobUrl, metricsURL, lastFailedRun.Format(time.RFC3339))))
+					}
+				}
+				if repoName == "" {
+					w.Write([]byte(fmt.Sprintf("plugin_status_since{name=%s,owner=%s,metricsUrl=%s} %s\n", Name, ownerName, metricsURL, statusSince.Format(time.RFC3339))))
+				} else {
+					w.Write([]byte(fmt.Sprintf("plugin_status_since{name=%s,owner=%s,repo=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, metricsURL, statusSince.Format(time.RFC3339))))
 				}
-				w.Write([]byte(fmt.Sprintf("plugin_status_since{name=%s,owner=%s,repo=%s,metricsUrl=%s} %s\n", Name, ownerName, repoName, metricsURL, statusSince.Format(time.RFC3339))))
 				if strings.Contains(pluginName, "_receiver") {
 					soloReceiver = true
 				} else {

internal/metrics/metrics.go

@@ -349,7 +349,9 @@ func (s *Server) handleJsonMetrics(ctx context.Context, soloReceiver bool) http.
 						case Plugin:
 							pluginMap["name"] = s.Name
 							pluginMap["plugin_name"] = s.PluginName
-							pluginMap["repo_name"] = s.RepoName
+							if s.RepoName != "" {
+								pluginMap["repo_name"] = s.RepoName
+							}
 							pluginMap["owner_name"] = s.OwnerName
 							pluginMap["status"] = s.Status
 							pluginMap["status_since"] = s.StatusSince
@@ -360,7 +362,9 @@ func (s *Server) handleJsonMetrics(ctx context.Context, soloReceiver bool) http.
 						case PluginBase:
 							pluginMap["name"] = s.Name
 							pluginMap["plugin_name"] = s.PluginName
-							pluginMap["repo_name"] = s.RepoName
+							if s.RepoName != "" {
+								pluginMap["repo_name"] = s.RepoName
+							}
 							pluginMap["owner_name"] = s.OwnerName
 							pluginMap["status"] = s.Status
 							pluginMap["status_since"] = s.StatusSince
@@ -410,7 +414,9 @@ func (s *Server) handleJsonMetrics(ctx context.Context, soloReceiver bool) http.
 						case Plugin:
 							pluginMap["name"] = s.Name
 							pluginMap["plugin_name"] = s.PluginName
-							pluginMap["repo_name"] = s.RepoName
+							if s.RepoName != "" {
+								pluginMap["repo_name"] = s.RepoName
+							}
 							pluginMap["owner_name"] = s.OwnerName
 							pluginMap["status"] = s.Status
 							pluginMap["status_since"] = s.StatusSince
@@ -421,7 +427,9 @@ func (s *Server) handleJsonMetrics(ctx context.Context, soloReceiver bool) http.
 						case PluginBase:
 							pluginMap["name"] = s.Name
 							pluginMap["plugin_name"] = s.PluginName
-							pluginMap["repo_name"] = s.RepoName
+							if s.RepoName != "" {
+								pluginMap["repo_name"] = s.RepoName
+							}
 							pluginMap["owner_name"] = s.OwnerName
 							pluginMap["status"] = s.Status
 							pluginMap["status_since"] = s.StatusSince
@@ -459,7 +467,9 @@ func (s *Server) handlePrometheusMetrics(ctx context.Context, soloReceiver bool)
 			case Plugin:
 				name = plugin.Name
 				pluginName = plugin.PluginName
-				repoName = plugin.RepoName
+				if plugin.RepoName != "" {
+					repoName = plugin.RepoName
+				}
 				ownerName = plugin.OwnerName
 				status = plugin.Status
 				StatusSince = plugin.StatusSince
@@ -470,19 +480,34 @@ func (s *Server) handlePrometheusMetrics(ctx context.Context, soloReceiver bool)
 			case PluginBase:
 				name = plugin.Name
 				pluginName = plugin.PluginName
-				repoName = plugin.RepoName
+				if plugin.RepoName != "" {
+					repoName = plugin.RepoName
+				}
 				ownerName = plugin.OwnerName
 				status = plugin.Status
 				StatusSince = plugin.StatusSince
 			default:
 				panic("unable to convert plugin to Plugin or PluginBase")
 			}
-			w.Write([]byte(fmt.Sprintf("plugin_status{name=%s,plugin=%s,owner=%s,repo=%s} %s\n", name, pluginName, ownerName, repoName, status)))
+			if repoName == "" {
+				w.Write([]byte(fmt.Sprintf("plugin_status{name=%s,plugin=%s,owner=%s} %s\n", name, pluginName, ownerName, status)))
+			} else {
+				w.Write([]byte(fmt.Sprintf("plugin_status{name=%s,plugin=%s,owner=%s,repo=%s} %s\n", name, pluginName, ownerName, repoName, status)))
+			}
 			if !strings.Contains(pluginName, "_receiver") {
-				w.Write([]byte(fmt.Sprintf("plugin_last_successful_run{name=%s,plugin=%s,owner=%s,repo=%s,job_url=%s} %s\n", name, pluginName, ownerName, repoName, lastSuccessfulRunJobUrl, lastSuccessfulRun.Format(time.RFC3339))))
-				w.Write([]byte(fmt.Sprintf("plugin_last_failed_run{name=%s,plugin=%s,owner=%s,repo=%s,job_url=%s} %s\n", name, pluginName, ownerName, repoName, lastFailedRunJobUrl, lastFailedRun.Format(time.RFC3339))))
+				if repoName == "" {
+					w.Write([]byte(fmt.Sprintf("plugin_last_successful_run{name=%s,plugin=%s,owner=%s,job_url=%s} %s\n", name, pluginName, ownerName, lastSuccessfulRunJobUrl, lastSuccessfulRun.Format(time.RFC3339))))
+					w.Write([]byte(fmt.Sprintf("plugin_last_failed_run{name=%s,plugin=%s,owner=%s,job_url=%s} %s\n", name, pluginName, ownerName, lastFailedRunJobUrl, lastFailedRun.Format(time.RFC3339))))
+				} else {
+					w.Write([]byte(fmt.Sprintf("plugin_last_successful_run{name=%s,plugin=%s,owner=%s,repo=%s,job_url=%s} %s\n", name, pluginName, ownerName, repoName, lastSuccessfulRunJobUrl, lastSuccessfulRun.Format(time.RFC3339))))
+					w.Write([]byte(fmt.Sprintf("plugin_last_failed_run{name=%s,plugin=%s,owner=%s,repo=%s,job_url=%s} %s\n", name, pluginName, ownerName, repoName, lastFailedRunJobUrl, lastFailedRun.Format(time.RFC3339))))
+				}
+			}
+			if repoName == "" {
+				w.Write([]byte(fmt.Sprintf("plugin_status_since{name=%s,plugin=%s,owner=%s} %s\n", name, pluginName, ownerName, StatusSince.Format(time.RFC3339))))
+			} else {
+				w.Write([]byte(fmt.Sprintf("plugin_status_since{name=%s,plugin=%s,owner=%s,repo=%s} %s\n", name, pluginName, ownerName, repoName, StatusSince.Format(time.RFC3339))))
 			}
-			w.Write([]byte(fmt.Sprintf("plugin_status_since{name=%s,plugin=%s,owner=%s,repo=%s} %s\n", name, pluginName, ownerName, repoName, StatusSince.Format(time.RFC3339))))
 		}
 		w.Write([]byte(fmt.Sprintf("host_cpu_count %d\n", metricsData.HostCPUCount)))
 		w.Write([]byte(fmt.Sprintf("host_cpu_used_count %d\n", metricsData.HostCPUUsedCount)))

main.go

@@ -71,11 +71,12 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+
 	var configPath string
+	var configFileName string
 	if *configFlag != "" {
 		configPath = *configFlag
 	} else {
-		var configFileName string
 		envConfigFileName := os.Getenv("ANKLET_CONFIG_FILE_NAME")
 		if envConfigFileName != "" {
 			configFileName = envConfigFileName
@@ -84,6 +85,7 @@ func main() {
 		}
 		configPath = filepath.Join(homeDir, ".config", "anklet", configFileName)
 	}
+	parentCtx = context.WithValue(parentCtx, config.ContextKey("configFileName"), configFileName)
 
 	// obtain config
 	loadedConfig, err := config.LoadConfig(configPath)
@@ -313,19 +315,31 @@ func worker(parentCtx context.Context, logger *slog.Logger, loadedConfig config.
 					panic("name is required for plugins")
 				}
 
+				if plugin.Repo == "" {
+					logger.InfoContext(pluginCtx, "no repo set for plugin; assuming it's an organization level plugin")
+					pluginCtx = context.WithValue(pluginCtx, config.ContextKey("isRepoSet"), false)
+				} else {
+					pluginCtx = context.WithValue(pluginCtx, config.ContextKey("isRepoSet"), true)
+				}
+
+				if plugin.PrivateKey == "" && loadedConfig.GlobalPrivateKey != "" {
+					plugin.PrivateKey = loadedConfig.GlobalPrivateKey
+				}
+
 				pluginCtx = context.WithValue(pluginCtx, config.ContextKey("plugin"), plugin)
 				pluginCtx = logging.AppendCtx(pluginCtx, slog.String("pluginName", plugin.Name))
 				pluginCtx = context.WithValue(pluginCtx, config.ContextKey("logger"), logger)
 
-				ankaCLI, err := anka.NewCLI(pluginCtx)
-				if err != nil {
-					pluginCancel()
-					logger.ErrorContext(pluginCtx, "unable to create anka cli", "error", err)
-					return
+				if !strings.Contains(plugin.Plugin, "_receiver") {
+					ankaCLI, err := anka.NewCLI(pluginCtx)
+					if err != nil {
+						pluginCancel()
+						logger.ErrorContext(pluginCtx, "unable to create anka cli", "error", err)
+						return
+					}
+					pluginCtx = context.WithValue(pluginCtx, config.ContextKey("ankacli"), ankaCLI)
 				}
 
-				pluginCtx = context.WithValue(pluginCtx, config.ContextKey("ankacli"), ankaCLI)
-
 				if plugin.Database.Enabled {
 					databaseClient, err := database.NewClient(pluginCtx, plugin.Database)
 					if err != nil {

plugins/handlers/github/README.md

@@ -18,9 +18,10 @@ plugins:
     # app_id: 12345678 # Org > Settings > Developer settings > GitHub Apps > New GitHub App
     # installation_id: 12345678 # You need to install the app (Org > Settings > Developer settings > GitHub Apps > click Edit for the app > Install App > select your Repo > then check the URL bar for the installation ID)
     registration: repo
-    repo: anklet
+    repo: anklet # Optional; only needed if registering a specific runner for a repo, otherwise it will be an org level runner.
     owner: veertuinc
     registry_url: http://anka.registry:8089
+    runner_group: macOS # requires Enterprise github
     # sleep_interval: 5 # Optional; defaults to 1 second.
     database:
       enabled: true
@@ -33,7 +34,8 @@ plugins:
 
 - Your PAT or Github App must have **Actions** and **Administration** Read & Write permissions.
 - The `database` is required. You can find installation instructions in the anklet main [README.md](../../README.md#database-setup).
-
+- If you are attempting to register runners for an entire organization, do NOT set `repo` and make sure your Github App has `Self-hosted runners` > `Read and write` permissions.
+- If your Organization level runner is registered and your public repo jobs are not picking it up even though the labels are a perfect match, make sure the Runner groups (likely `Default`) has `Allow public repositories`.
 
 Next, in your workflow yml you need to add several labels to `runs-on`. Here is the list and an example: