Traefik v2 integration

K3s upgrade via watch over file change of static file and manifest
and triggers helm-controller for change. It seems reasonable to
only allow upgrade traefik v1->v2 when there is no existing custom
traefik HelmChartConfig in the cluster to avoid any
incompatibility.

Here also separate the CRDs and put them into a different chart
to support CRD upgrade.

Signed-off-by: Chin-Ya Huang <chin-ya.huang@suse.com>
This commit is contained in:
Chin-Ya Huang 2020-12-23 09:28:19 +08:00 committed by Erik Wilson
parent f970e49b7d
commit 10e0328977
No known key found for this signature in database
GPG Key ID: 28E43BB8BE202CF8
13 changed files with 249 additions and 45 deletions

View File

@ -32,6 +32,9 @@ RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.30.0; \
fi
ENV YQ_URL=https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux
RUN wget -O - ${YQ_URL}_$(go env GOARCH) > /usr/bin/yq && chmod +x /usr/bin/yq
ARG SELINUX=true
ENV SELINUX $SELINUX

View File

@ -0,0 +1,7 @@
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: traefik-crd
namespace: kube-system
spec:
chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-9.12.3.tgz

View File

@ -4,20 +4,24 @@ metadata:
name: traefik
namespace: kube-system
spec:
chart: https://%{KUBERNETES_API}%/static/charts/traefik-1.81.0.tgz
chart: https://%{KUBERNETES_API}%/static/charts/traefik-9.12.3.tgz
valuesContent: |-
rbac:
enabled: true
ssl:
ports:
websecure:
tls:
enabled: true
metrics:
prometheus:
podAnnotations:
prometheus.io/port: "8082"
prometheus.io/scrape: "true"
providers:
kubernetesIngress:
publishedService:
enabled: true
kubernetes:
ingressEndpoint:
useDefaultPublishedService: true
priorityClassName: "system-cluster-critical"
image: "rancher/library-traefik"
image:
name: "traefik"
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"

View File

@ -6,5 +6,5 @@ const (
// coredns and servicelb run controllers that are turned off when their manifests are disabled.
// The k3s CloudController also has a bundled manifest and can be disabled via the
// --disable-cloud-controller flag or --disable=ccm, but the latter method is not documented.
DisableItems = "coredns, servicelb, traefik, local-storage, metrics-server"
DisableItems = "coredns, servicelb, traefik, traefik-crd, local-storage, metrics-server"
)

View File

@ -11,6 +11,7 @@
// manifests/metrics-server/metrics-server-service.yaml
// manifests/metrics-server/resource-reader.yaml
// manifests/rolebindings.yaml
// manifests/traefik-crd.yaml
// manifests/traefik.yaml
// +build !no_stage
@ -310,7 +311,27 @@ func rolebindingsYaml() (*asset, error) {
return a, nil
}
var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x92\xcf\x8a\xdb\x4c\x10\xc4\xef\x7a\x8a\x46\xb0\x47\x49\x9f\x6f\x1f\x73\xdb\x38\x86\x84\x80\xb3\xc4\x49\xae\xa1\x35\x2a\x5b\x83\x47\x33\xa2\xbb\x65\xa2\xfc\x79\xf7\x20\xad\xe3\x0d\x21\x81\x25\xba\xa9\xe7\x57\xd5\x35\xc5\xf0\x18\x3e\x42\x34\xe4\xe4\xa8\x47\x1c\x6a\xcf\x66\x11\x75\xc8\xcd\x65\x53\x9c\x43\xea\x1c\xbd\x42\x1c\xb6\x3d\x8b\x15\x03\x8c\x3b\x36\x76\x05\x51\xe2\x01\x8e\x4c\x18\xc7\x70\xbe\xfe\xeb\xc8\x1e\x8e\xce\x53\x8b\x4a\x67\x35\x0c\x85\x8e\xf0\x0b\xee\x17\x03\x47\xbd\xd9\xa8\xae\x69\xee\xbe\xbe\xf9\xf0\x62\xf7\x6e\xbf\x7b\xbf\x3b\x7c\xba\x7f\x78\xfd\xfd\xae\x51\x63\x0b\xbe\x59\x41\x6d\xae\xc6\xd5\xa6\xfe\x7f\x53\xff\x57\xdb\xe9\x4b\x41\x74\xe1\x38\x41\xb7\x39\x19\x92\x39\xfa\x56\x15\x44\x44\xd2\xf2\xba\x62\xf9\x90\xb8\x8d\xe8\x96\x60\x13\xd6\x99\x6a\xfc\xfb\xe1\x00\x93\xe0\xf5\x27\x30\x4a\x1e\x60\x3d\xa6\xdb\xe4\x4f\xa2\xe5\x7a\x92\x60\xb8\x51\x21\x9d\x04\xaa\xbb\xd4\x8d\x39\x24\x7b\x12\x4f\x8a\x97\x38\xf2\x14\xed\x61\x6a\x63\xd0\x1e\xdd\x01\x72\x09\x1e\xbf\xf8\x8d\x12\xb2\x04\x9b\xb7\x91\x55\xf7\x6b\xad\xe5\x63\x7b\x95\x8f\x93\x1a\xa4\xf2\x12\x2c\x78\x8e\xe5\x2a\x08\x03\x9f\x16\x48\x38\xf9\x1e\xd2\xc4\xd0\x0a\xcb\x5c\x5d\x3b\x7b\x84\x2c\x47\x08\x5b\xc8\xe9\x16\xb3\xa2\x33\x66\x47\xe5\xf6\xea\x76\xdf\x75\x39\xe9\xdb\x14\xe7\xf2\x96\x38\x8f\x8b\x2a\x8b\xa3\x72\xf7\x39\xa8\x69\xf9\x9b\x38\xe5\x0e\x95\xe4\x88\xfa\xa9\x87\xe5\xb9\xf8\x9c\x4c\x72\xac\xc6\xc8\x09\xcf\xf0\x23\xc2\xf1\x08\x6f\x8e\xca\x7d\x3e\xf8\x1e\xdd\x14\xf1\xdc\x65\x03\x2f\xb5\xfc\xfb\x96\x1f\x01\x00\x00\xff\xff\x8e\x38\x58\x4b\xf7\x02\x00\x00")
var _traefikCrdYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4c\xcc\x3d\x0b\xc2\x30\x10\xc6\xf1\x3d\x9f\x22\x4b\xc7\x26\x54\x27\x6f\x53\x29\x28\x82\x88\x6f\xab\x9c\xe9\x69\x43\x9b\x36\xe4\x4e\x41\xc5\xef\x2e\x15\x07\xd7\x87\xe7\xff\xc3\xe8\x8f\x94\xd8\xf7\x1d\xe8\x9a\xda\x60\x1c\x8a\xb4\x64\x7c\x6f\xef\x85\x6a\x7c\x57\x81\x5e\x50\x1b\xe6\x35\x26\x51\x81\x04\x2b\x14\x04\xa5\x75\x87\x81\x40\x4b\x42\xba\xf8\x26\x77\xa9\xfa\x6d\x1c\xd1\x11\xe8\xe6\x76\xa6\x9c\x1f\x2c\x14\x14\x47\x72\x43\xe2\x06\x04\x74\x2d\x12\x19\xac\xcd\x5e\xab\xc3\xac\xdc\xae\xcb\x7d\xb9\x3b\x4d\x37\xcb\x77\x66\x59\x50\xbc\xb3\xdf\x23\xdb\x3f\x3c\x9f\x98\x62\x64\xc6\x46\xae\x4f\xf5\x09\x00\x00\xff\xff\xc3\x85\xab\xf0\xb4\x00\x00\x00")
func traefikCrdYamlBytes() ([]byte, error) {
return bindataRead(
_traefikCrdYaml,
"traefik-crd.yaml",
)
}
func traefikCrdYaml() (*asset, error) {
bytes, err := traefikCrdYamlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "traefik-crd.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x91\x4d\x6f\xdb\x3c\x10\x84\xef\xfa\x15\x0b\x01\x39\x4a\x7a\x93\xf7\x92\xf2\xe6\x1a\x06\x1a\x14\x70\x8b\xba\xed\xb5\xa0\xa9\xb1\x45\x98\x22\x89\xdd\x95\x5b\xf7\xe3\xbf\x17\xf4\x57\x52\x20\x40\x8a\xea\x46\x6a\xf8\xec\xec\x8c\xcd\xfe\x33\x58\x7c\x8a\x86\x06\x84\xb1\x75\x56\x35\xa0\xf5\xa9\xdb\xdf\x56\x3b\x1f\x7b\x43\x6f\x10\xc6\xf9\x60\x59\xab\x11\x6a\x7b\xab\xd6\x54\x44\xd1\x8e\x30\xa4\x6c\xb1\xf1\xbb\xf3\x59\xb2\x75\x30\xb4\x9b\xd6\x68\xe4\x20\x8a\xb1\x92\x0c\x57\xe4\xae\x00\x0c\x0d\xaa\x59\x4c\xd7\xdd\xfc\x78\xfb\xe9\xf5\xe2\xc3\x72\xf1\x71\xb1\xfa\x32\x7b\xff\xf0\xeb\xa6\x13\xb5\xea\x5d\x77\x14\x4a\x77\x06\x37\xaf\xda\xdb\xbb\xf6\xff\x56\xb7\xdf\x2b\xa2\xbd\x0d\x13\x64\x9e\xa2\x22\xaa\xa1\x9f\x4d\x45\x44\xc4\x6b\x7b\x1c\x51\x3e\x44\xbb\x0e\xe8\x8b\xb1\x09\xc7\xbb\x9c\x58\xe5\xf2\xfb\x2b\xd6\x02\x37\x31\x2e\x17\x44\x1a\xe4\xf1\xf0\x3c\xa0\x9f\xc5\x98\x8a\xbb\x14\xaf\xda\xcc\x69\x84\x0e\x98\xa4\x64\x55\x86\x18\xaa\xef\xff\xbb\xbf\xab\x9f\x15\x88\x63\x9b\x61\xa8\x2e\xd8\x93\x24\x73\xda\xfb\x1e\x7c\x45\x96\xd8\x38\x42\x21\x0f\x71\xcb\x90\x27\xbe\xf2\xb4\x0e\x5e\x06\xf4\x2b\xf0\xde\x3b\xbc\xe0\x98\x7d\x62\xaf\x87\x79\xb0\x22\xcb\x63\x4f\xf5\xa9\x8e\xc6\x85\x49\x14\xdc\x38\xf6\xea\x9d\x0d\x27\x2b\x7e\xb4\xdb\x2b\xf3\x54\x6c\x7d\x2e\xe0\x24\xd0\x14\xc0\x4f\x03\x68\x68\x87\x83\xa1\x7a\x7e\xe6\xcc\xfa\x3e\x45\x79\x17\xc3\xe1\xb2\x7f\xca\xe5\x45\x62\x43\xf5\xe2\x9b\x17\x95\xfa\x8f\x87\x31\xf5\x68\x38\x05\xb4\x8f\x6b\x97\xa0\x5c\x8a\xca\x29\x34\x39\xd8\x88\x17\x58\x44\xd8\x6c\xe0\x4a\xf2\xcb\xb4\x72\x03\xfa\x29\xe0\xef\xc6\x8c\xb6\xc4\xf0\x6f\xfc\xdf\x01\x00\x00\xff\xff\x49\x78\x8d\x66\x34\x03\x00\x00")
func traefikYamlBytes() ([]byte, error) {
return bindataRead(
@ -393,6 +414,7 @@ var _bindata = map[string]func() (*asset, error){
"metrics-server/metrics-server-service.yaml": metricsServerMetricsServerServiceYaml,
"metrics-server/resource-reader.yaml": metricsServerResourceReaderYaml,
"rolebindings.yaml": rolebindingsYaml,
"traefik-crd.yaml": traefikCrdYaml,
"traefik.yaml": traefikYaml,
}
@ -450,6 +472,7 @@ var _bintree = &bintree{nil, map[string]*bintree{
"resource-reader.yaml": &bintree{metricsServerResourceReaderYaml, map[string]*bintree{}},
}},
"rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}},
"traefik-crd.yaml": &bintree{traefikCrdYaml, map[string]*bintree{}},
"traefik.yaml": &bintree{traefikYaml, map[string]*bintree{}},
}}

View File

@ -8,6 +8,7 @@ import (
"io/ioutil"
net2 "net"
"os"
"path"
"path/filepath"
"strconv"
"strings"
@ -203,10 +204,10 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control) error {
dataDir := filepath.Join(controlConfig.DataDir, "static")
if err := static.Stage(dataDir); err != nil {
stageTraefik := doStageTraefik(sc)
if err := static.Stage(dataDir, stageTraefik); err != nil {
return err
}
dataDir = filepath.Join(controlConfig.DataDir, "manifests")
templateVars := map[string]string{
"%{CLUSTER_DNS}%": controlConfig.ClusterDNS.String(),
@ -214,13 +215,58 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
"%{DEFAULT_LOCAL_STORAGE_PATH}%": controlConfig.DefaultLocalStoragePath,
}
if err := deploy.Stage(dataDir, templateVars, controlConfig.Skips); err != nil {
skip := controlConfig.Skips
if !stageTraefik {
skip["traefik"] = true
skip["traefik-crd"] = true
}
if err := deploy.Stage(dataDir, templateVars, skip); err != nil {
return err
}
return deploy.WatchFiles(ctx, sc.Apply, sc.K3s.K3s().V1().Addon(), controlConfig.Disables, dataDir)
}
// doStageTraefik checks on running traefik HelmChart version and traefik
// HelmChartConfig.
// Traefik should skip stage when it is v1 and have existing customize traefik
// HelmChartConfig due to the incompatible configuration from v1 to v2.
// It will progress stage on upgrade or restart when no customized traefik
// HelmChartConfig exists on the cluster.
func doStageTraefik(sc *Context) bool {
if isHelmChartTraefikV1(sc) && isHelmChartConfigExist(sc, "traefik") {
return false
}
return true
}
// isHelmChartTraefikV1 checks the chart with "traefik-1." prefix.
func isHelmChartTraefikV1(sc *Context) bool {
prefix := "traefik-1."
helmChart, err := sc.Helm.Helm().V1().HelmChart().Get(metav1.NamespaceSystem, "traefik", metav1.GetOptions{})
if err != nil {
logrus.WithError(err).Info("Not find traefik")
return false
}
chart := path.Base(helmChart.Spec.Chart)
if strings.HasPrefix(chart, prefix) {
logrus.WithField("chart", chart).Info("Found traefik v1 running")
return true
}
return false
}
func isHelmChartConfigExist(sc *Context, name string) bool {
helmChartConfig := sc.Helm.Helm().V1().HelmChartConfig()
_, err := helmChartConfig.Get(metav1.NamespaceSystem, name, metav1.GetOptions{})
if err != nil {
logrus.WithField("name", name).Info("Not find HelmChartConfig")
return false
}
logrus.WithField("name", name).Info("Found HelmChartConfig ")
return true
}
func HomeKubeConfig(write, rootless bool) (string, error) {
if write {
if os.Getuid() == 0 && !rootless {

View File

@ -6,13 +6,18 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
func Stage(dataDir string) error {
func Stage(dataDir string, isStageTraefik bool) error {
for _, name := range AssetNames() {
if !isStageTraefik && strings.HasPrefix(name, "charts/traefik-") {
logrus.WithField("name", name).Info("Skip staging")
continue
}
content, err := Asset(name)
if err != nil {
return err

File diff suppressed because one or more lines are too long

View File

@ -1,8 +1,8 @@
docker.io/library/traefik:2.3.6
docker.io/rancher/coredns-coredns:1.8.0
docker.io/rancher/klipper-helm:v0.4.3
docker.io/rancher/klipper-lb:v0.1.2
docker.io/rancher/library-busybox:1.32.1
docker.io/rancher/library-traefik:1.7.19
docker.io/rancher/local-path-provisioner:v0.0.19
docker.io/rancher/metrics-server:v0.3.6
docker.io/rancher/pause:3.1

View File

@ -0,0 +1,7 @@
apiVersion: ${api_version}
version: ${chart_version}
description: Installs the CRDs for ${name}.
name: ${name}-crd
type: application
annotations:
catalog.cattle.io/hidden: "true"

View File

@ -0,0 +1,2 @@
# ${name}-crd
A Rancher chart that installs the CRDs used by [${name}](https://github.com/rancher/dev-charts/tree/master/packages/${name}).

View File

@ -0,0 +1,14 @@
#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}}
# {{- \$found := dict -}}
${set_found_crd}
# {{- range .Capabilities.APIVersions -}}
# {{- if hasKey \$found (toString .) -}}
# {{- set \$found (toString .) true -}}
# {{- end -}}
# {{- end -}}
# {{- range \$_, \$exists := \$found -}}
# {{- if (eq \$exists false) -}}
# {{- required "Required CRDs are missing. Please install the ${name}-crd chart before installing this chart." "" -}}
# {{- end -}}
# {{- end -}}
#{{- end -}}

View File

@ -5,23 +5,93 @@ cd $(dirname $0)/..
. ./scripts/version.sh
ROOT_VERSION=v0.8.1
TRAEFIK_VERSION=1.81.0
TRAEFIK_VERSION=9.12.3 # appVersion: 2.3.6
CHARTS_DIR=build/static/charts
mkdir -p ${CHARTS_DIR}
curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${ROOT_VERSION}/k3s-root-${ARCH}.tar | tar xf -
TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz
TRAEFIK_URL=https://charts.helm.sh/stable/packages/${TRAEFIK_FILE}
setup_tmp() {
TMP_DIR=$(mktemp -d --tmpdir=${CHARTS_DIR})
cleanup() {
code=$?
set +e
trap - EXIT
rm -rf ${TMP_DIR}
exit $code
}
trap cleanup INT EXIT
}
echo "Downloading Traefik Helm chart from ${TRAEFIK_URL}"
curl -sfL ${TRAEFIK_URL} -o ${CHARTS_DIR}/${TRAEFIK_FILE}
code=$?
download_and_package_traefik () {
echo "Downloading Traefik Helm chart from ${TRAEFIK_URL}"
curl -sfL ${TRAEFIK_URL} -o ${TMP_DIR}/${TRAEFIK_FILE}
code=$?
if [ $code -ne 0 ]; then
if [ $code -ne 0 ]; then
echo "Error: Failed to download Traefik Helm chart!"
exit $code
fi
fi
echo "Uncompress ${TMP_DIR}/${TRAEFIK_FILE}"
tar xf ${TMP_DIR}/${TRAEFIK_FILE} -C ${TMP_DIR}
echo "Prepare traefik CRD"
TRAEFIK_TMP_CHART=${TMP_DIR}/traefik
TRAEFIK_TMP_CRD=${TRAEFIK_TMP_CHART}-crd
# Collect information on chart
name=$(cat ${TRAEFIK_TMP_CHART}/Chart.yaml | yq r - 'name')
api_version=$(cat ${TRAEFIK_TMP_CHART}/Chart.yaml | yq r - 'apiVersion')
chart_version=$(cat ${TRAEFIK_TMP_CHART}/Chart.yaml | yq r - 'version')
# Collect information on CRDs
crd_apis=()
for crd_yaml in ${TRAEFIK_TMP_CHART}/crds/*; do
crd_group=$(yq r ${crd_yaml} 'spec.group')
crd_kind=$(yq r ${crd_yaml} 'spec.names.kind')
crd_version=$(yq r ${crd_yaml} 'spec.version')
if [[ -z "$crd_version" ]]; then
crd_version=$(yq r ${crd_yaml} 'spec.versions[0].name')
fi
crd_apis+=("${crd_group}/${crd_version}/${crd_kind}")
done
set_found_crd=$(
for crd in ${crd_apis[@]}; do
echo "# {{- set \$found \"${crd}\" false -}}"
done
)
# Copy base template and apply variables to the template
mkdir -p ${TRAEFIK_TMP_CRD}
cp -R ./scripts/chart-templates/crd-base/* ${TRAEFIK_TMP_CRD}
for template_file in $(find ${TRAEFIK_TMP_CRD} -type f); do
# Applies any environment variables currently set onto your template file
eval "echo \"$(sed 's/"/\\"/g' ${template_file})\"" > ${template_file}
done
# Move anything from ${f}/charts-crd/overlay-upstream to the main chart
cp -R ${TRAEFIK_TMP_CRD}/overlay-upstream/* ${TRAEFIK_TMP_CHART}
rm -rf ${TRAEFIK_TMP_CRD}/overlay-upstream
# Move CRDs from main chart to CRD chart
mkdir -p ${TRAEFIK_TMP_CRD}/templates
mv ${TRAEFIK_TMP_CHART}/crds/* ${TRAEFIK_TMP_CRD}/templates
rm -rf ${TRAEFIK_TMP_CHART}/crds
# Package charts
OPTS="--format gnu --sort=name --owner=0 --group=0 --numeric-owner"
GZIP=-n tar ${OPTS} --mtime='UTC 2021-01-01' -cz -f ${CHARTS_DIR}/${TRAEFIK_FILE} -C ${TMP_DIR} $(basename ${TRAEFIK_TMP_CHART})
GZIP=-n tar ${OPTS} --mtime='UTC 2021-01-01' -cz -f ${CHARTS_DIR}/${TRAEFIK_CRD_FILE} -C ${TMP_DIR} $(basename ${TRAEFIK_TMP_CRD})
}
TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz
TRAEFIK_CRD_FILE=traefik-crd-${TRAEFIK_VERSION}.tgz
TRAEFIK_URL=https://helm.traefik.io/traefik/${TRAEFIK_FILE}
setup_tmp
download_and_package_traefik
cp scripts/wg-add.sh bin/aux/