From 1d666d9515a8d2ab37a623008872b86b69776c2d Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 4 Feb 2019 16:33:23 -0700 Subject: [PATCH] Add helm controller --- manifests/nginx-ingress.yaml | 7 + pkg/helm/controller.go | 309 ++++++++++++++++++++++++++ pkg/server/server.go | 7 + types/apis/k3s.cattle.io/v1/schema.go | 5 +- types/apis/k3s.cattle.io/v1/types.go | 24 ++ types/codegen/main.go | 32 ++- 6 files changed, 374 insertions(+), 10 deletions(-) create mode 100644 manifests/nginx-ingress.yaml create mode 100644 pkg/helm/controller.go diff --git a/manifests/nginx-ingress.yaml b/manifests/nginx-ingress.yaml new file mode 100644 index 0000000000..e27da8e4ac --- /dev/null +++ b/manifests/nginx-ingress.yaml @@ -0,0 +1,7 @@ +apiVersion: k3s.cattle.io/v1 +kind: HelmChart +metadata: + name: nginx-ingress + namespace: kube-system +spec: + chart: stable/nginx-ingress diff --git a/pkg/helm/controller.go b/pkg/helm/controller.go new file mode 100644 index 0000000000..bb52ac2676 --- /dev/null +++ b/pkg/helm/controller.go @@ -0,0 +1,309 @@ +package helm + +import ( + "context" + "fmt" + "sort" + + batchclient "github.com/rancher/k3s/types/apis/batch/v1" + coreclient "github.com/rancher/k3s/types/apis/core/v1" + k3s "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1" + rbacclients "github.com/rancher/k3s/types/apis/rbac.authorization.k8s.io/v1" + "github.com/rancher/norman/pkg/changeset" + "github.com/rancher/norman/pkg/objectset" + batch "k8s.io/api/batch/v1" + core "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + namespace = "kube-system" + image = "rancher/klipper-helm:v0.1.0" + label = "helm.k3s.cattle.io/chart" +) + +var ( + trueVal = true +) + +func Register(ctx context.Context) error { + k3sClients := k3s.ClientsFrom(ctx) + coreClients := coreclient.ClientsFrom(ctx) + jobClients := batchclient.ClientsFrom(ctx) + rbacClients := rbacclients.ClientsFrom(ctx) + + h := &handler{ + jobs: jobClients.Job, + jobCache: jobClients.Job.Cache(), + processor: objectset.NewProcessor("k3s.helm"). + Client(coreClients.ConfigMap). + Client(coreClients.ServiceAccount). + Client(jobClients.Job). + Client(rbacClients.ClusterRoleBinding). + Patcher(batch.SchemeGroupVersion.WithKind("Job"), objectset.ReplaceOnChange), + } + + k3sClients.HelmChart.OnChange(ctx, "helm", h.onChange) + k3sClients.HelmChart.OnRemove(ctx, "helm", h.onRemove) + + changeset.Watch(ctx, "helm-pod-watch", + func(namespace, name string, obj runtime.Object) ([]changeset.Key, error) { + if job, ok := obj.(*batch.Job); ok { + name := job.Labels[label] + if name != "" { + return []changeset.Key{ + { + Name: name, + Namespace: namespace, + }, + }, nil + } + } + return nil, nil + }, + k3sClients.HelmChart, + jobClients.Job) + return nil +} + +type handler struct { + jobCache batchclient.JobClientCache + jobs batchclient.JobClient + processor *objectset.Processor +} + +func (h *handler) onChange(chart *k3s.HelmChart) (runtime.Object, error) { + if chart.Namespace != namespace || chart.Spec.Chart == "" { + return chart, nil + } + + objs := objectset.NewObjectSet() + job, configMap := job(chart) + objs.Add(serviceAccount(chart)) + objs.Add(roleBinding(chart)) + objs.Add(job) + if configMap != nil { + objs.Add(configMap) + } + + if err := h.processor.NewDesiredSet(chart, objs).Apply(); err != nil { + return chart, err + } + + chart.Status.JobName = job.Name + return chart, nil +} + +func (h *handler) onRemove(chart *k3s.HelmChart) (runtime.Object, error) { + if chart.Namespace != namespace || chart.Spec.Chart == "" { + return chart, nil + } + + job, _ := job(chart) + + job, err := h.jobCache.Get(chart.Namespace, job.Name) + if errors.IsNotFound(err) { + _, err := h.onChange(chart) + if err != nil { + return chart, err + } + } else if err != nil { + return nil, err + } + + if job.Status.Succeeded <= 0 { + return nil, fmt.Errorf("waiting for delete of helm chart %s", chart.Name) + } + + return chart, h.processor.NewDesiredSet(chart, objectset.NewObjectSet()).Apply() +} + +func job(chart *k3s.HelmChart) (*batch.Job, *core.ConfigMap) { + action := "install" + if chart.DeletionTimestamp != nil { + action = "delete" + } + job := &batch.Job{ + TypeMeta: meta.TypeMeta{ + APIVersion: "batch/v1", + Kind: "Job", + }, + ObjectMeta: meta.ObjectMeta{ + Name: fmt.Sprintf("helm-%s-%s", action, chart.Name), + Namespace: chart.Namespace, + Labels: map[string]string{ + label: chart.Name, + }, + }, + Spec: batch.JobSpec{ + Template: core.PodTemplateSpec{ + ObjectMeta: meta.ObjectMeta{ + Labels: map[string]string{ + label: chart.Name, + }, + }, + Spec: core.PodSpec{ + RestartPolicy: core.RestartPolicyOnFailure, + Containers: []core.Container{ + { + Name: "helm", + Image: image, + ImagePullPolicy: core.PullAlways, + Args: args(chart), + Env: []core.EnvVar{ + { + Name: "NAME", + Value: chart.Name, + }, + { + Name: "VERSION", + Value: chart.Spec.Version, + }, + { + Name: "REPO", + Value: chart.Spec.Repo, + }, + }, + }, + }, + ServiceAccountName: fmt.Sprintf("helm-%s", chart.Name), + }, + }, + }, + } + + configMap := configMap(chart) + if configMap == nil { + return job, nil + } + + job.Spec.Template.Spec.Volumes = []core.Volume{ + { + Name: "values", + VolumeSource: core.VolumeSource{ + ConfigMap: &core.ConfigMapVolumeSource{ + LocalObjectReference: core.LocalObjectReference{ + Name: configMap.Name, + }, + }, + }, + }, + } + + job.Spec.Template.Spec.Containers[0].VolumeMounts = []core.VolumeMount{ + { + MountPath: "/config", + Name: "values", + }, + } + + return job, configMap +} + +func configMap(chart *k3s.HelmChart) *core.ConfigMap { + if chart.Spec.ValuesContent == "" { + return nil + } + + return &core.ConfigMap{ + TypeMeta: meta.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: meta.ObjectMeta{ + Name: fmt.Sprintf("chart-values-%s", chart.Name), + Namespace: chart.Namespace, + }, + Data: map[string]string{ + "values.yaml": chart.Spec.ValuesContent, + }, + } +} + +func roleBinding(chart *k3s.HelmChart) *rbac.ClusterRoleBinding { + return &rbac.ClusterRoleBinding{ + TypeMeta: meta.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: meta.ObjectMeta{ + Name: fmt.Sprintf("helm-%s-%s", chart.Namespace, chart.Name), + }, + RoleRef: rbac.RoleRef{ + Kind: "ClusterRole", + APIGroup: "rbac.authorization.k8s.io", + Name: "cluster-admin", + }, + Subjects: []rbac.Subject{ + { + Name: fmt.Sprintf("helm-%s", chart.Name), + Kind: "ServiceAccount", + Namespace: chart.Namespace, + }, + }, + } + +} + +func serviceAccount(chart *k3s.HelmChart) *core.ServiceAccount { + return &core.ServiceAccount{ + TypeMeta: meta.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: meta.ObjectMeta{ + Name: fmt.Sprintf("helm-%s", chart.Name), + Namespace: chart.Namespace, + }, + AutomountServiceAccountToken: &trueVal, + } +} + +func args(chart *k3s.HelmChart) []string { + if chart.DeletionTimestamp != nil { + return []string{ + "delete", + "--purge", chart.Name, + } + } + + spec := chart.Spec + args := []string{ + "install", + "--name", chart.Name, + spec.Chart, + } + if spec.TargetNamespace != "" { + args = append(args, "--namespace", spec.TargetNamespace) + } + if spec.Repo != "" { + args = append(args, "--repo", spec.Repo) + } + if spec.Version != "" { + args = append(args, "--version", spec.Version) + } + + for _, k := range keys(spec.Set) { + val := spec.Set[k] + if val.StrVal != "" { + args = append(args, "--set-string", fmt.Sprintf("%s=%s", k, val.StrVal)) + } else { + args = append(args, "--set", fmt.Sprintf("%s=%d", k, val.IntVal)) + } + } + + return args +} + +func keys(val map[string]intstr.IntOrString) []string { + var keys []string + for k := range val { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 6c562223ab..907eec911f 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -17,11 +17,14 @@ import ( "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/daemons/control" "github.com/rancher/k3s/pkg/deploy" + "github.com/rancher/k3s/pkg/helm" "github.com/rancher/k3s/pkg/servicelb" "github.com/rancher/k3s/pkg/tls" appsv1 "github.com/rancher/k3s/types/apis/apps/v1" + batchv1 "github.com/rancher/k3s/types/apis/batch/v1" corev1 "github.com/rancher/k3s/types/apis/core/v1" v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1" + rbacv1 "github.com/rancher/k3s/types/apis/rbac.authorization.k8s.io/v1" "github.com/rancher/norman" "github.com/rancher/norman/pkg/clientaccess" "github.com/rancher/norman/pkg/dynamiclistener" @@ -91,6 +94,8 @@ func startNorman(ctx context.Context, config *Config) (string, error) { v1.Factory, appsv1.Factory, corev1.Factory, + batchv1.Factory, + rbacv1.Factory, }, Schemas: []*types.Schemas{ v1.Schemas, @@ -99,6 +104,7 @@ func startNorman(ctx context.Context, config *Config) (string, error) { &v1.APIVersion: { v1.ListenerConfigGroupVersionKind.Kind, v1.AddonGroupVersionKind.Kind, + v1.HelmChartGroupVersionKind.Kind, }, }, IgnoredKubeConfigEnv: true, @@ -108,6 +114,7 @@ func startNorman(ctx context.Context, config *Config) (string, error) { }, DisableLeaderElection: true, MasterControllers: []norman.ControllerRegister{ + helm.Register, func(ctx context.Context) error { return servicelb.Register(ctx, norman.GetServer(ctx).K8sClient, !config.DisableServiceLB) }, diff --git a/types/apis/k3s.cattle.io/v1/schema.go b/types/apis/k3s.cattle.io/v1/schema.go index 38f872bd9b..0da5e105ce 100644 --- a/types/apis/k3s.cattle.io/v1/schema.go +++ b/types/apis/k3s.cattle.io/v1/schema.go @@ -13,6 +13,7 @@ var ( } Schemas = factory.Schemas(&APIVersion). - MustImport(&APIVersion, ListenerConfig{}). - MustImport(&APIVersion, Addon{}) + MustImport(&APIVersion, Addon{}). + MustImport(&APIVersion, HelmChart{}). + MustImport(&APIVersion, ListenerConfig{}) ) diff --git a/types/apis/k3s.cattle.io/v1/types.go b/types/apis/k3s.cattle.io/v1/types.go index 54c3d78459..1a900b1ffd 100644 --- a/types/apis/k3s.cattle.io/v1/types.go +++ b/types/apis/k3s.cattle.io/v1/types.go @@ -5,6 +5,7 @@ import ( "github.com/rancher/norman/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" ) type ListenerConfig struct { @@ -34,3 +35,26 @@ type AddonSpec struct { type AddonStatus struct { GVKs []schema.GroupVersionKind `json:"gvks,omitempty"` } + +type HelmChart struct { + types.Namespaced + + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec HelmChartSpec `json:"spec,omitempty"` + Status HelmChartStatus `json:"status,omitempty"` +} + +type HelmChartSpec struct { + TargetNamespace string `json:"targetNamespace,omitempty"` + Chart string `json:"chart,omitempty"` + Version string `json:"version,omitempty"` + Repo string `json:"repo,omitempty"` + Set map[string]intstr.IntOrString `json:"set,omitempty"` + ValuesContent string `json:"valuesContent,omitempty"` +} + +type HelmChartStatus struct { + JobName string `json:"jobName,omitempty"` +} diff --git a/types/codegen/main.go b/types/codegen/main.go index be6e2164a8..3f11edf655 100644 --- a/types/codegen/main.go +++ b/types/codegen/main.go @@ -5,8 +5,10 @@ import ( v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1" "github.com/rancher/norman/generator" "github.com/sirupsen/logrus" - v13 "k8s.io/api/apps/v1" - v12 "k8s.io/api/core/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" ) var ( @@ -68,18 +70,32 @@ func main() { logrus.Fatal(err) } - if err := generator.ControllersForForeignTypes(basePackage, v12.SchemeGroupVersion, []interface{}{ - v12.Service{}, - v12.Pod{}, + if err := generator.ControllersForForeignTypes(basePackage, corev1.SchemeGroupVersion, []interface{}{ + corev1.ServiceAccount{}, + corev1.Service{}, + corev1.Pod{}, + corev1.ConfigMap{}, }, []interface{}{ - v12.Node{}, + corev1.Node{}, }); err != nil { logrus.Fatal(err) } - if err := generator.ControllersForForeignTypes(basePackage, v13.SchemeGroupVersion, []interface{}{ - v13.Deployment{}, + if err := generator.ControllersForForeignTypes(basePackage, appsv1.SchemeGroupVersion, []interface{}{ + appsv1.Deployment{}, }, nil); err != nil { logrus.Fatal(err) } + + if err := generator.ControllersForForeignTypes(basePackage, batchv1.SchemeGroupVersion, []interface{}{ + batchv1.Job{}, + }, nil); err != nil { + logrus.Fatal(err) + } + + if err := generator.ControllersForForeignTypes(basePackage, rbacv1.SchemeGroupVersion, nil, []interface{}{ + rbacv1.ClusterRoleBinding{}, + }); err != nil { + logrus.Fatal(err) + } }