Skip to content

Commit

Permalink
Adds support for reading AutoscalingRunnerSet githubConfigSecret from…
Browse files Browse the repository at this point in the history
… controller namespace
  • Loading branch information
lacarvalho91 committed Aug 22, 2024
1 parent 1be410b commit a86e981
Show file tree
Hide file tree
Showing 9 changed files with 193 additions and 79 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ spec:
{{- with .Values.flags.updateStrategy }}
- "--update-strategy={{ . }}"
{{- end }}
{{- if .Values.flags.readGitHubConfigSecretFromControllerNamespace }}
- "--read-github-config-secret-from-controller-namespace"
{{- end }}
{{- if .Values.metrics }}
{{- with .Values.metrics }}
- "--listener-metrics-addr={{ .listenerAddr }}"
Expand Down
6 changes: 6 additions & 0 deletions charts/gha-runner-scale-set-controller/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,12 @@ flags:
## that you don't have any overprovisioning of runners.
updateStrategy: "immediate"

## This flag determines whether the githubConfigSecret on the AutoscalingRunnerSet will be read from the
## namespace that the controller is running in or the namespace that the AutoscalingRunnerSet resource is in.
##
## Defaults to false, so the secret will be read from the namespace that the runner lives in.
readGitHubConfigSecretFromControllerNamespace: false

## Defines a list of prefixes that should not be propagated to internal resources.
## This is useful when you have labels that are used for internal purposes and should not be propagated to internal resources.
## See https://github.com/actions/actions-runner-controller/issues/3533 for more information.
Expand Down
15 changes: 12 additions & 3 deletions controllers/actions.github.com/autoscalinglistener_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,9 @@ const (
// AutoscalingListenerReconciler reconciles a AutoscalingListener object
type AutoscalingListenerReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
Log logr.Logger
Scheme *runtime.Scheme
ReadGitHubConfigSecretFromControllerNamespace bool
// ListenerMetricsAddr is address that the metrics endpoint binds to.
// If it is set to "0", the metrics server is not started.
ListenerMetricsAddr string
Expand Down Expand Up @@ -130,7 +131,7 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl.

// Check if the GitHub config secret exists
secret := new(corev1.Secret)
if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace, Name: autoscalingListener.Spec.GitHubConfigSecret}, secret); err != nil {
if err := r.Get(ctx, types.NamespacedName{Namespace: r.deriveConfigSecretNamespace(autoscalingListener), Name: autoscalingListener.Spec.GitHubConfigSecret}, secret); err != nil {
log.Error(err, "Failed to find GitHub config secret.",
"namespace", autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
"name", autoscalingListener.Spec.GitHubConfigSecret)
Expand Down Expand Up @@ -275,6 +276,14 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl.
return ctrl.Result{}, nil
}

func (r *AutoscalingListenerReconciler) deriveConfigSecretNamespace(autoscalingListener *v1alpha1.AutoscalingListener) string {
secretNamespace := autoscalingListener.Spec.AutoscalingRunnerSetNamespace
if r.ReadGitHubConfigSecretFromControllerNamespace {
secretNamespace = autoscalingListener.Namespace
}
return secretNamespace
}

func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, autoscalingListener *v1alpha1.AutoscalingListener, logger logr.Logger) (done bool, err error) {
logger.Info("Cleaning up the listener pod")
listenerPod := new(corev1.Pod)
Expand Down
13 changes: 11 additions & 2 deletions controllers/actions.github.com/autoscalingrunnerset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type AutoscalingRunnerSetReconciler struct {
Log logr.Logger
Scheme *runtime.Scheme
ControllerNamespace string
ReadGitHubConfigSecretFromControllerNamespace bool
DefaultRunnerScaleSetListenerImage string
DefaultRunnerScaleSetListenerImagePullSecrets []string
UpdateStrategy UpdateStrategy
Expand Down Expand Up @@ -208,7 +209,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
}

secret := new(corev1.Secret)
if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: autoscalingRunnerSet.Spec.GitHubConfigSecret}, secret); err != nil {
if err := r.Get(ctx, types.NamespacedName{Namespace: r.deriveConfigSecretNamespace(autoscalingRunnerSet), Name: autoscalingRunnerSet.Spec.GitHubConfigSecret}, secret); err != nil {
log.Error(err, "Failed to find GitHub config secret.",
"namespace", autoscalingRunnerSet.Namespace,
"name", autoscalingRunnerSet.Spec.GitHubConfigSecret)
Expand Down Expand Up @@ -678,7 +679,7 @@ func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Con

func (r *AutoscalingRunnerSetReconciler) actionsClientFor(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (actions.ActionsService, error) {
var configSecret corev1.Secret
if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: autoscalingRunnerSet.Spec.GitHubConfigSecret}, &configSecret); err != nil {
if err := r.Get(ctx, types.NamespacedName{Namespace: r.deriveConfigSecretNamespace(autoscalingRunnerSet), Name: autoscalingRunnerSet.Spec.GitHubConfigSecret}, &configSecret); err != nil {
return nil, fmt.Errorf("failed to find GitHub config secret: %w", err)
}

Expand All @@ -696,6 +697,14 @@ func (r *AutoscalingRunnerSetReconciler) actionsClientFor(ctx context.Context, a
)
}

func (r *AutoscalingRunnerSetReconciler) deriveConfigSecretNamespace(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) string {
secretNamespace := autoscalingRunnerSet.Namespace
if r.ReadGitHubConfigSecretFromControllerNamespace {
secretNamespace = r.ControllerNamespace
}
return secretNamespace
}

func (r *AutoscalingRunnerSetReconciler) actionsClientOptionsFor(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) ([]actions.ClientOption, error) {
var options []actions.ClientOption

Expand Down
103 changes: 76 additions & 27 deletions controllers/actions.github.com/autoscalingrunnerset_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
var mgr ctrl.Manager
var controller *AutoscalingRunnerSetReconciler
var autoscalingNS *corev1.Namespace
var controllerNS *corev1.Namespace
var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet
var configSecret *corev1.Secret

Expand All @@ -61,7 +62,11 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {

BeforeEach(func() {
ctx = context.Background()
autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient)
var namespaces []*corev1.Namespace
namespaces, mgr = createNamespaces(GinkgoT(), 2)
autoscalingNS = namespaces[0]
controllerNS = namespaces[1]

configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name)

controller = &AutoscalingRunnerSetReconciler{
Expand All @@ -77,32 +82,7 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {

min := 1
max := 10
autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
autoscalingRunnerSet = createAutoscalingRunnerSet("test-asrs", autoscalingNS, buildVersion, configSecret.Name, min, max)

err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
Expand Down Expand Up @@ -197,6 +177,46 @@ var _ = Describe("Test AutoScalingRunnerSet controller", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet")
Expect(len(runnerSetList.Items)).To(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created")
})

It("It should read GitHub config secret from controller namespace if ReadGitHubConfigSecretFromControllerNamespace is enabled", func() {
err := k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: configSecret.Name, Namespace: autoscalingNS.Name}})
Expect(err).ToNot(HaveOccurred(), "failed to delete github-config-secret")

controller2 := controller
controller2.ReadGitHubConfigSecretFromControllerNamespace = true
controller2.ControllerNamespace = controllerNS.Name

err = controller2.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")

controllerNamespaceConfigSecret := createDefaultSecret(GinkgoT(), k8sClient, controller2.ControllerNamespace)
runner := createAutoscalingRunnerSet("secret-test-asrs", autoscalingNS, buildVersion, controllerNamespaceConfigSecret.Name, 0, 1)
err = k8sClient.Create(ctx, runner)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoscalingRunnerSet")

created := new(v1alpha1.AutoscalingRunnerSet)
// Check if runner scale set is created on service
Eventually(
func() (string, error) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: runner.Name, Namespace: autoscalingRunnerSet.Namespace}, created)
if err != nil {
return "", err
}

if _, ok := created.Annotations[runnerScaleSetIdAnnotationKey]; !ok {
return "", nil
}

if _, ok := created.Annotations[AnnotationKeyGitHubRunnerGroupName]; !ok {
return "", nil
}

return fmt.Sprintf("%s_%s", created.Annotations[runnerScaleSetIdAnnotationKey], created.Annotations[AnnotationKeyGitHubRunnerGroupName]), nil
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval).Should(BeEquivalentTo("1_testgroup"), "RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's annotation")

})
})

Context("When deleting a new AutoScalingRunnerSet", func() {
Expand Down Expand Up @@ -1790,3 +1810,32 @@ var _ = Describe("Test resource version and build version mismatch", func() {
).Should(BeTrue())
})
})

func createAutoscalingRunnerSet(name string, ns *corev1.Namespace, buildVersion string, secretName string, min, max int) *v1alpha1.AutoscalingRunnerSet {
return &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns.Name,
Labels: map[string]string{
LabelKeyKubernetesVersion: buildVersion,
},
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: secretName,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
}
14 changes: 13 additions & 1 deletion controllers/actions.github.com/ephemeralrunner_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ type EphemeralRunnerReconciler struct {
Log logr.Logger
Scheme *runtime.Scheme
ActionsClient actions.MultiClient

ControllerNamespace string
ReadGitHubConfigSecretFromControllerNamespace bool

ResourceBuilder
}

Expand Down Expand Up @@ -592,6 +596,14 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
return ctrl.Result{}, nil
}

func (r *EphemeralRunnerReconciler) deriveConfigSecretNamespace(ephemeralRunner *v1alpha1.EphemeralRunner) string {
secretNamespace := ephemeralRunner.Namespace
if r.ReadGitHubConfigSecretFromControllerNamespace {
secretNamespace = r.ControllerNamespace
}
return secretNamespace
}

func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, log logr.Logger) (ctrl.Result, error) {
var envs []corev1.EnvVar
if runner.Spec.ProxySecretRef != "" {
Expand Down Expand Up @@ -712,7 +724,7 @@ func (r *EphemeralRunnerReconciler) updateRunStatusFromPod(ctx context.Context,

func (r *EphemeralRunnerReconciler) actionsClientFor(ctx context.Context, runner *v1alpha1.EphemeralRunner) (actions.ActionsService, error) {
secret := new(corev1.Secret)
if err := r.Get(ctx, types.NamespacedName{Namespace: runner.Namespace, Name: runner.Spec.GitHubConfigSecret}, secret); err != nil {
if err := r.Get(ctx, types.NamespacedName{Namespace: r.deriveConfigSecretNamespace(runner), Name: runner.Spec.GitHubConfigSecret}, secret); err != nil {
return nil, fmt.Errorf("failed to get secret: %w", err)
}

Expand Down
14 changes: 12 additions & 2 deletions controllers/actions.github.com/ephemeralrunnerset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ type EphemeralRunnerSetReconciler struct {
Scheme *runtime.Scheme
ActionsClient actions.MultiClient

PublishMetrics bool
PublishMetrics bool
ControllerNamespace string
ReadGitHubConfigSecretFromControllerNamespace bool

ResourceBuilder
}
Expand Down Expand Up @@ -504,7 +506,7 @@ func (r *EphemeralRunnerSetReconciler) deleteEphemeralRunnerWithActionsClient(ct

func (r *EphemeralRunnerSetReconciler) actionsClientFor(ctx context.Context, rs *v1alpha1.EphemeralRunnerSet) (actions.ActionsService, error) {
secret := new(corev1.Secret)
if err := r.Get(ctx, types.NamespacedName{Namespace: rs.Namespace, Name: rs.Spec.EphemeralRunnerSpec.GitHubConfigSecret}, secret); err != nil {
if err := r.Get(ctx, types.NamespacedName{Namespace: r.deriveConfigSecretNamespace(rs), Name: rs.Spec.EphemeralRunnerSpec.GitHubConfigSecret}, secret); err != nil {
return nil, fmt.Errorf("failed to get secret: %w", err)
}

Expand All @@ -522,6 +524,14 @@ func (r *EphemeralRunnerSetReconciler) actionsClientFor(ctx context.Context, rs
)
}

func (r *EphemeralRunnerSetReconciler) deriveConfigSecretNamespace(rs *v1alpha1.EphemeralRunnerSet) string {
secretNamespace := rs.Namespace
if r.ReadGitHubConfigSecretFromControllerNamespace {
secretNamespace = r.ControllerNamespace
}
return secretNamespace
}

func (r *EphemeralRunnerSetReconciler) actionsClientOptionsFor(ctx context.Context, rs *v1alpha1.EphemeralRunnerSet) ([]actions.ClientOption, error) {
var opts []actions.ClientOption
if rs.Spec.EphemeralRunnerSpec.Proxy != nil {
Expand Down
40 changes: 24 additions & 16 deletions controllers/actions.github.com/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,29 +35,37 @@ func startManagers(t ginkgo.GinkgoTInterface, first manager.Manager, others ...m
}
}

func createNamespace(t ginkgo.GinkgoTInterface, client client.Client) (*corev1.Namespace, manager.Manager) {
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling" + RandStringRunes(5)},
}

err := k8sClient.Create(context.Background(), ns)
require.NoError(t, err)
func createNamespaces(t ginkgo.GinkgoTInterface, numNamespaces int) ([]*corev1.Namespace, manager.Manager) {
namespaces := make([]*corev1.Namespace, 0)
defaultNamespaces := make(map[string]cache.Config)
for range numNamespaces {
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling" + RandStringRunes(5)},
}
namespaces = append(namespaces, ns)

t.Cleanup(func() {
err := k8sClient.Delete(context.Background(), ns)
err := k8sClient.Create(context.Background(), ns)
require.NoError(t, err)
})

t.Cleanup(func() {
err := k8sClient.Delete(context.Background(), ns)
require.NoError(t, err)
})

defaultNamespaces[ns.Name] = cache.Config{}
}

mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Cache: cache.Options{
DefaultNamespaces: map[string]cache.Config{
ns.Name: {},
},
},
Cache: cache.Options{DefaultNamespaces: defaultNamespaces},
})
require.NoError(t, err)

return ns, mgr
return namespaces, mgr
}

func createNamespace(t ginkgo.GinkgoTInterface, client client.Client) (*corev1.Namespace, manager.Manager) {
namespaces, mgr := createNamespaces(t, 1)
return namespaces[0], mgr
}

func createDefaultSecret(t ginkgo.GinkgoTInterface, client client.Client, namespace string) *corev1.Secret {
Expand Down
Loading

0 comments on commit a86e981

Please sign in to comment.