diff --git a/load_testing/k3s/.gitignore b/load_testing/k3s/.gitignore new file mode 100644 index 0000000000..e79eb23105 --- /dev/null +++ b/load_testing/k3s/.gitignore @@ -0,0 +1 @@ +.terraform* diff --git a/load_testing/k3s/cluster-loader/.gitignore b/load_testing/k3s/cluster-loader/.gitignore new file mode 100644 index 0000000000..ee5ab6892d --- /dev/null +++ b/load_testing/k3s/cluster-loader/.gitignore @@ -0,0 +1 @@ +kubeConfig.yaml diff --git a/load_testing/k3s/cluster-loader/large/config.yaml b/load_testing/k3s/cluster-loader/large/config.yaml new file mode 100644 index 0000000000..2b9f23f23a --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/config.yaml @@ -0,0 +1,471 @@ +# ASSUMPTIONS: +# - Underlying cluster should have 100+ nodes. +# - Number of nodes should be divisible by NODES_PER_NAMESPACE (default 100). +# - The number of created SVCs is half the number of created Deployments. +# - Only half of Deployments will be assigned 1-1 to existing SVCs. + +#Constants +{{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}} +{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 100}} +{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 30}} +{{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}} +{{$BIG_GROUP_SIZE := 25}} +{{$MEDIUM_GROUP_SIZE := 15}} +{{$SMALL_GROUP_SIZE := 1}} +{{$SMALL_STATEFUL_SETS_PER_NAMESPACE := 1}} +{{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE := 1}} +{{$ENABLE_CHAOSMONKEY := DefaultParam .ENABLE_CHAOSMONKEY false}} +{{$ENABLE_PROMETHEUS_API_RESPONSIVENESS := DefaultParam .ENABLE_PROMETHEUS_API_RESPONSIVENESS false}} +{{$ENABLE_CONFIGMAPS := DefaultParam .ENABLE_CONFIGMAPS false}} +{{$ENABLE_SECRETS := DefaultParam .ENABLE_SECRETS false}} +{{$ENABLE_STATEFULSETS := DefaultParam .ENABLE_STATEFULSETS false}} +#Variables +{{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE}} +{{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}} +{{$podsPerNamespace := DivideInt $totalPods $namespaces}} +{{$saturationTime := DivideInt $totalPods $LOAD_TEST_THROUGHPUT}} +# bigDeployments - 1/4 of namespace pods should be in big Deployments. +{{$bigDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $BIG_GROUP_SIZE)}} +# mediumDeployments - 1/4 of namespace pods should be in medium Deployments. +{{$mediumDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $MEDIUM_GROUP_SIZE)}} +# smallDeployments - 1/2 of namespace pods should be in small Deployments. +{{$smallDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 2 $SMALL_GROUP_SIZE)}} +# If StatefulSets are enabled reduce the number of small and medium deployments per namespace +{{$smallDeploymentsPerNamespace := SubtractInt $smallDeploymentsPerNamespace (IfThenElse $ENABLE_STATEFULSETS $SMALL_STATEFUL_SETS_PER_NAMESPACE 0)}} +{{$mediumDeploymentsPerNamespace := SubtractInt $mediumDeploymentsPerNamespace (IfThenElse $ENABLE_STATEFULSETS $MEDIUM_STATEFUL_SETS_PER_NAMESPACE 0)}} + + +name: load +automanagedNamespaces: {{$namespaces}} +tuningSets: +- name: Sequence + parallelismLimitedLoad: + parallelismLimit: 1 +- name: RandomizedSaturationTimeLimited + RandomizedTimeLimitedLoad: + timeLimit: {{$saturationTime}}s +- name: RandomizedScalingTimeLimited + RandomizedTimeLimitedLoad: + # The expected number of created/deleted pods is totalPods/4 when scaling, + # as each RS changes its size from X to a uniform random value in [X/2, 3X/2]. + # To match 10 [pods/s] requirement, we need to divide saturationTime by 4. + timeLimit: {{DivideInt $saturationTime 4}}s +{{if $ENABLE_CHAOSMONKEY}} +chaosMonkey: + nodeFailure: + failureRate: 0.01 + interval: 1m + jitterFactor: 10.0 + simulatedDowntime: 10m +{{end}} +steps: +- name: Starting measurements + measurements: + - Identifier: APIResponsiveness + Method: APIResponsiveness + Params: + action: reset + - Identifier: APIResponsivenessPrometheus + Method: APIResponsivenessPrometheus + Params: + action: start + - Identifier: PodStartupLatency + Method: PodStartupLatency + Params: + action: start + labelSelector: group = load + threshold: 1h + - Identifier: InClusterNetworkLatency + Method: InClusterNetworkLatency + Params: + action: start + replicasPerProbe: {{DivideInt .Nodes 100}} + - Identifier: DnsLookupLatency + Method: DnsLookupLatency + Params: + action: start + replicasPerProbe: {{DivideInt .Nodes 100}} + - Identifier: NetworkProgrammingLatency + Method: NetworkProgrammingLatency + Params: + action: start + - Identifier: TestMetrics + Method: TestMetrics + Params: + action: start + nodeMode: {{$NODE_MODE}} + +- name: Creating SVCs + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{DivideInt (AddInt $bigDeploymentsPerNamespace 1) 2}} + tuningSet: Sequence + objectBundle: + - basename: big-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{DivideInt (AddInt $mediumDeploymentsPerNamespace 1) 2}} + tuningSet: Sequence + objectBundle: + - basename: medium-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{DivideInt (AddInt $smallDeploymentsPerNamespace 1) 2}} + tuningSet: Sequence + objectBundle: + - basename: small-service + objectTemplatePath: service.yaml + +- name: Starting measurement for waiting for pods + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: start + apiVersion: apps/v1 + kind: Deployment + labelSelector: group = load + operationTimeout: 15m + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: start + apiVersion: apps/v1 + kind: StatefulSet + labelSelector: group = load + operationTimeout: 15m + {{end}} + +- name: Creating objects + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$bigDeploymentsPerNamespace}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + {{if $ENABLE_CONFIGMAPS}} + - basename: big-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: big-deployment + objectTemplatePath: secret.yaml + {{end}} + - basename: big-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{$BIG_GROUP_SIZE}} + ReplicasMax: {{$BIG_GROUP_SIZE}} + SvcName: big-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$mediumDeploymentsPerNamespace}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + {{if $ENABLE_CONFIGMAPS}} + - basename: medium-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: medium-deployment + objectTemplatePath: secret.yaml + {{end}} + - basename: medium-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{$MEDIUM_GROUP_SIZE}} + ReplicasMax: {{$MEDIUM_GROUP_SIZE}} + SvcName: medium-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$smallDeploymentsPerNamespace}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + {{if $ENABLE_CONFIGMAPS}} + - basename: small-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: small-deployment + objectTemplatePath: secret.yaml + {{end}} + - basename: small-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{$SMALL_GROUP_SIZE}} + ReplicasMax: {{$SMALL_GROUP_SIZE}} + SvcName: small-service + {{if $ENABLE_STATEFULSETS}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: small-statefulset + objectTemplatePath: statefulset_service.yaml + - basename: small-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{$SMALL_GROUP_SIZE}} + ReplicasMax: {{$SMALL_GROUP_SIZE}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: medium-statefulset + objectTemplatePath: statefulset_service.yaml + - basename: medium-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{$MEDIUM_GROUP_SIZE}} + ReplicasMax: {{$MEDIUM_GROUP_SIZE}} + {{end}} + +- name: Waiting for pods to be running + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: gather + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: gather + {{end}} + +- name: Scaling objects + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$bigDeploymentsPerNamespace}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: big-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $BIG_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $BIG_GROUP_SIZE 1.5}} + SvcName: big-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$mediumDeploymentsPerNamespace}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: medium-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $MEDIUM_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $MEDIUM_GROUP_SIZE 1.5}} + SvcName: medium-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$smallDeploymentsPerNamespace}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: small-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $SMALL_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $SMALL_GROUP_SIZE 1.5}} + SvcName: small-service + {{if $ENABLE_STATEFULSETS}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: small-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $SMALL_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $SMALL_GROUP_SIZE 1.5}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: medium-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $MEDIUM_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $MEDIUM_GROUP_SIZE 1.5}} + {{end}} + +- name: Waiting for objects to become scaled + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: gather + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: gather + {{end}} + +- name: Deleting objects + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: big-deployment + objectTemplatePath: deployment.yaml + {{if $ENABLE_CONFIGMAPS}} + - basename: big-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: big-deployment + objectTemplatePath: secret.yaml + {{end}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: medium-deployment + objectTemplatePath: deployment.yaml + {{if $ENABLE_CONFIGMAPS}} + - basename: medium-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: medium-deployment + objectTemplatePath: secret.yaml + {{end}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: small-deployment + objectTemplatePath: deployment.yaml + {{if $ENABLE_CONFIGMAPS}} + - basename: small-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: small-deployment + objectTemplatePath: secret.yaml + {{end}} + {{if $ENABLE_STATEFULSETS}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: small-statefulset + objectTemplatePath: statefulset.yaml + - basename: small-statefulset + objectTemplatePath: statefulset_service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: medium-statefulset + objectTemplatePath: statefulset.yaml + - basename: medium-statefulset + objectTemplatePath: statefulset_service.yaml + {{end}} + +- name: Waiting for pods to be deleted + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: gather + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: gather + {{end}} + +- name: Deleting SVCs + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: Sequence + objectBundle: + - basename: big-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: Sequence + objectBundle: + - basename: medium-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: Sequence + objectBundle: + - basename: small-service + objectTemplatePath: service.yaml + +- name: Collecting measurements + measurements: + - Identifier: APIResponsiveness + Method: APIResponsiveness + Params: + action: gather + - Identifier: APIResponsivenessPrometheus + Method: APIResponsivenessPrometheus + Params: + action: gather + {{if $ENABLE_PROMETHEUS_API_RESPONSIVENESS}} + enableViolations: true + {{end}} + - Identifier: PodStartupLatency + Method: PodStartupLatency + Params: + action: gather + - Identifier: InClusterNetworkLatency + Method: InClusterNetworkLatency + Params: + action: gather + - Identifier: DnsLookupLatency + Method: DnsLookupLatency + Params: + action: gather + - Identifier: NetworkProgrammingLatency + Method: NetworkProgrammingLatency + Params: + action: gather + - Identifier: TestMetrics + Method: TestMetrics + Params: + action: gather diff --git a/load_testing/k3s/cluster-loader/large/configmap.yaml b/load_testing/k3s/cluster-loader/large/configmap.yaml new file mode 100644 index 0000000000..b249a39143 --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.Name}} +data: + data.yaml: |- + a: 1 + b: 2 + c: 3 diff --git a/load_testing/k3s/cluster-loader/large/deployment.yaml b/load_testing/k3s/cluster-loader/large/deployment.yaml new file mode 100644 index 0000000000..dcd581914a --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/deployment.yaml @@ -0,0 +1,62 @@ +{{$EnableConfigMaps := DefaultParam .ENABLE_CONFIGMAPS false}} +{{$EnableSecrets := DefaultParam .ENABLE_SECRETS false}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.Name}} + labels: + group: load + svc: {{.SvcName}}-{{.Index}} +spec: + replicas: {{RandIntRange .ReplicasMin .ReplicasMax}} + selector: + matchLabels: + name: {{.Name}} + template: + metadata: + labels: + group: load + name: {{.Name}} + svc: {{.SvcName}}-{{.Index}} + spec: + containers: + - image: k8s.gcr.io/pause:3.1 + name: {{.Name}} + resources: + requests: + cpu: 10m + memory: "10M" + volumeMounts: + {{if and $EnableConfigMaps (eq (Mod .Index 20) 0 19) }} # .Index % 20 in {0,19} - 10% deployments will have ConfigMap + - name: configmap + mountPath: /var/configmap + {{end}} + {{if and $EnableSecrets (eq (Mod .Index 20) 10 19) }} # .Index % 20 in {10,19} - 10% deployments will have Secret + - name: secret + mountPath: /var/secret + {{end}} + dnsPolicy: Default + terminationGracePeriodSeconds: 1 + # Add not-ready/unreachable tolerations for 15 minutes so that node + # failure doesn't trigger pod deletion. + tolerations: + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 900 + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 900 + volumes: + {{if and $EnableConfigMaps (eq (Mod .Index 20) 0 19) }} # .Index % 20 in {0,19} - 10% deployments will have ConfigMap + - name: configmap + configMap: + name: {{.BaseName}}-{{.Index}} + {{end}} + {{if and $EnableSecrets (eq (Mod .Index 20) 10 19) }} # .Index % 20 in {10,19} - 10% deployments will have Secret + - name: secret + secret: + secretName: {{.BaseName}}-{{.Index}} + {{end}} diff --git a/load_testing/k3s/cluster-loader/large/run-test.sh b/load_testing/k3s/cluster-loader/large/run-test.sh new file mode 100644 index 0000000000..b190c99ef3 --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/run-test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +clusterloader --kubeconfig=../kubeConfig.yaml --testconfig=config.yaml diff --git a/load_testing/k3s/cluster-loader/large/secret.yaml b/load_testing/k3s/cluster-loader/large/secret.yaml new file mode 100644 index 0000000000..67134b355f --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} +type: Opaque +data: + password: c2NhbGFiaWxpdHkK diff --git a/load_testing/k3s/cluster-loader/large/service.yaml b/load_testing/k3s/cluster-loader/large/service.yaml new file mode 100644 index 0000000000..ed6a22c8cf --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/service.yaml @@ -0,0 +1,16 @@ +{{$SetServiceProxyLabel := DefaultParam .SetServiceProxyLabel false}} + +apiVersion: v1 +kind: Service +metadata: + name: {{.Name}} +{{if and $SetServiceProxyLabel (eq (Mod .Index 2) 0)}} + labels: + service.kubernetes.io/service-proxy-name: foo +{{end}} +spec: + selector: + svc: {{.Name}} + ports: + - port: 80 + targetPort: 80 diff --git a/load_testing/k3s/cluster-loader/large/statefulset.yaml b/load_testing/k3s/cluster-loader/large/statefulset.yaml new file mode 100644 index 0000000000..bb97bfce9a --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/statefulset.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{.Name}} + labels: + group: load +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + name: {{.Name}} + serviceName: {{.Name}} + replicas: {{RandIntRange .ReplicasMin .ReplicasMax}} + template: + metadata: + labels: + group: statefulset + name: {{.Name}} + spec: + terminationGracePeriodSeconds: 1 + containers: + - name: {{.Name}} + image: k8s.gcr.io/pause:3.1 + ports: + - containerPort: 80 + name: web + resources: + requests: + cpu: 10m + memory: "10M" diff --git a/load_testing/k3s/cluster-loader/large/statefulset_service.yaml b/load_testing/k3s/cluster-loader/large/statefulset_service.yaml new file mode 100644 index 0000000000..5e16a47a19 --- /dev/null +++ b/load_testing/k3s/cluster-loader/large/statefulset_service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.Name}} + labels: + name: {{.Name}} +spec: + clusterIP: None + selector: + name: {{.Name}} diff --git a/load_testing/k3s/cluster-loader/small/config.yaml b/load_testing/k3s/cluster-loader/small/config.yaml new file mode 100644 index 0000000000..3c999b5069 --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/config.yaml @@ -0,0 +1,471 @@ +# ASSUMPTIONS: +# - Underlying cluster should have 100+ nodes. +# - Number of nodes should be divisible by NODES_PER_NAMESPACE (default 100). +# - The number of created SVCs is half the number of created Deployments. +# - Only half of Deployments will be assigned 1-1 to existing SVCs. + +#Constants +{{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}} +{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 1}} +{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 5}} +{{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}} +{{$BIG_GROUP_SIZE := 25}} +{{$MEDIUM_GROUP_SIZE := 15}} +{{$SMALL_GROUP_SIZE := 1}} +{{$SMALL_STATEFUL_SETS_PER_NAMESPACE := 1}} +{{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE := 1}} +{{$ENABLE_CHAOSMONKEY := DefaultParam .ENABLE_CHAOSMONKEY false}} +{{$ENABLE_PROMETHEUS_API_RESPONSIVENESS := DefaultParam .ENABLE_PROMETHEUS_API_RESPONSIVENESS false}} +{{$ENABLE_CONFIGMAPS := DefaultParam .ENABLE_CONFIGMAPS false}} +{{$ENABLE_SECRETS := DefaultParam .ENABLE_SECRETS false}} +{{$ENABLE_STATEFULSETS := DefaultParam .ENABLE_STATEFULSETS false}} +#Variables +{{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE}} +{{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}} +{{$podsPerNamespace := DivideInt $totalPods $namespaces}} +{{$saturationTime := DivideInt $totalPods $LOAD_TEST_THROUGHPUT}} +# bigDeployments - 1/4 of namespace pods should be in big Deployments. +{{$bigDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $BIG_GROUP_SIZE)}} +# mediumDeployments - 1/4 of namespace pods should be in medium Deployments. +{{$mediumDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 4 $MEDIUM_GROUP_SIZE)}} +# smallDeployments - 1/2 of namespace pods should be in small Deployments. +{{$smallDeploymentsPerNamespace := DivideInt $podsPerNamespace (MultiplyInt 2 $SMALL_GROUP_SIZE)}} +# If StatefulSets are enabled reduce the number of small and medium deployments per namespace +{{$smallDeploymentsPerNamespace := SubtractInt $smallDeploymentsPerNamespace (IfThenElse $ENABLE_STATEFULSETS $SMALL_STATEFUL_SETS_PER_NAMESPACE 0)}} +{{$mediumDeploymentsPerNamespace := SubtractInt $mediumDeploymentsPerNamespace (IfThenElse $ENABLE_STATEFULSETS $MEDIUM_STATEFUL_SETS_PER_NAMESPACE 0)}} + + +name: load +automanagedNamespaces: {{$namespaces}} +tuningSets: +- name: Sequence + parallelismLimitedLoad: + parallelismLimit: 1 +- name: RandomizedSaturationTimeLimited + RandomizedTimeLimitedLoad: + timeLimit: {{$saturationTime}}s +- name: RandomizedScalingTimeLimited + RandomizedTimeLimitedLoad: + # The expected number of created/deleted pods is totalPods/4 when scaling, + # as each RS changes its size from X to a uniform random value in [X/2, 3X/2]. + # To match 10 [pods/s] requirement, we need to divide saturationTime by 4. + timeLimit: {{DivideInt $saturationTime 4}}s +{{if $ENABLE_CHAOSMONKEY}} +chaosMonkey: + nodeFailure: + failureRate: 0.01 + interval: 1m + jitterFactor: 10.0 + simulatedDowntime: 10m +{{end}} +steps: +- name: Starting measurements + measurements: + - Identifier: APIResponsiveness + Method: APIResponsiveness + Params: + action: reset + - Identifier: APIResponsivenessPrometheus + Method: APIResponsivenessPrometheus + Params: + action: start + - Identifier: PodStartupLatency + Method: PodStartupLatency + Params: + action: start + labelSelector: group = load + threshold: 1h + - Identifier: InClusterNetworkLatency + Method: InClusterNetworkLatency + Params: + action: start + replicasPerProbe: {{DivideInt .Nodes 100}} + - Identifier: DnsLookupLatency + Method: DnsLookupLatency + Params: + action: start + replicasPerProbe: {{DivideInt .Nodes 100}} + - Identifier: NetworkProgrammingLatency + Method: NetworkProgrammingLatency + Params: + action: start + - Identifier: TestMetrics + Method: TestMetrics + Params: + action: start + nodeMode: {{$NODE_MODE}} + +- name: Creating SVCs + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{DivideInt (AddInt $bigDeploymentsPerNamespace 1) 2}} + tuningSet: Sequence + objectBundle: + - basename: big-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{DivideInt (AddInt $mediumDeploymentsPerNamespace 1) 2}} + tuningSet: Sequence + objectBundle: + - basename: medium-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{DivideInt (AddInt $smallDeploymentsPerNamespace 1) 2}} + tuningSet: Sequence + objectBundle: + - basename: small-service + objectTemplatePath: service.yaml + +- name: Starting measurement for waiting for pods + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: start + apiVersion: apps/v1 + kind: Deployment + labelSelector: group = load + operationTimeout: 15m + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: start + apiVersion: apps/v1 + kind: StatefulSet + labelSelector: group = load + operationTimeout: 15m + {{end}} + +- name: Creating objects + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$bigDeploymentsPerNamespace}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + {{if $ENABLE_CONFIGMAPS}} + - basename: big-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: big-deployment + objectTemplatePath: secret.yaml + {{end}} + - basename: big-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{$BIG_GROUP_SIZE}} + ReplicasMax: {{$BIG_GROUP_SIZE}} + SvcName: big-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$mediumDeploymentsPerNamespace}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + {{if $ENABLE_CONFIGMAPS}} + - basename: medium-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: medium-deployment + objectTemplatePath: secret.yaml + {{end}} + - basename: medium-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{$MEDIUM_GROUP_SIZE}} + ReplicasMax: {{$MEDIUM_GROUP_SIZE}} + SvcName: medium-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$smallDeploymentsPerNamespace}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + {{if $ENABLE_CONFIGMAPS}} + - basename: small-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: small-deployment + objectTemplatePath: secret.yaml + {{end}} + - basename: small-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{$SMALL_GROUP_SIZE}} + ReplicasMax: {{$SMALL_GROUP_SIZE}} + SvcName: small-service + {{if $ENABLE_STATEFULSETS}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: small-statefulset + objectTemplatePath: statefulset_service.yaml + - basename: small-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{$SMALL_GROUP_SIZE}} + ReplicasMax: {{$SMALL_GROUP_SIZE}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: medium-statefulset + objectTemplatePath: statefulset_service.yaml + - basename: medium-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{$MEDIUM_GROUP_SIZE}} + ReplicasMax: {{$MEDIUM_GROUP_SIZE}} + {{end}} + +- name: Waiting for pods to be running + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: gather + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: gather + {{end}} + +- name: Scaling objects + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$bigDeploymentsPerNamespace}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: big-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $BIG_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $BIG_GROUP_SIZE 1.5}} + SvcName: big-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$mediumDeploymentsPerNamespace}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: medium-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $MEDIUM_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $MEDIUM_GROUP_SIZE 1.5}} + SvcName: medium-service + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$smallDeploymentsPerNamespace}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: small-deployment + objectTemplatePath: deployment.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $SMALL_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $SMALL_GROUP_SIZE 1.5}} + SvcName: small-service + {{if $ENABLE_STATEFULSETS}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: small-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $SMALL_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $SMALL_GROUP_SIZE 1.5}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}} + tuningSet: RandomizedScalingTimeLimited + objectBundle: + - basename: medium-statefulset + objectTemplatePath: statefulset.yaml + templateFillMap: + ReplicasMin: {{MultiplyInt $MEDIUM_GROUP_SIZE 0.5}} + ReplicasMax: {{MultiplyInt $MEDIUM_GROUP_SIZE 1.5}} + {{end}} + +- name: Waiting for objects to become scaled + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: gather + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: gather + {{end}} + +- name: Deleting objects + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: big-deployment + objectTemplatePath: deployment.yaml + {{if $ENABLE_CONFIGMAPS}} + - basename: big-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: big-deployment + objectTemplatePath: secret.yaml + {{end}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: medium-deployment + objectTemplatePath: deployment.yaml + {{if $ENABLE_CONFIGMAPS}} + - basename: medium-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: medium-deployment + objectTemplatePath: secret.yaml + {{end}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: small-deployment + objectTemplatePath: deployment.yaml + {{if $ENABLE_CONFIGMAPS}} + - basename: small-deployment + objectTemplatePath: configmap.yaml + {{end}} + {{if $ENABLE_SECRETS}} + - basename: small-deployment + objectTemplatePath: secret.yaml + {{end}} + {{if $ENABLE_STATEFULSETS}} + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: small-statefulset + objectTemplatePath: statefulset.yaml + - basename: small-statefulset + objectTemplatePath: statefulset_service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: RandomizedSaturationTimeLimited + objectBundle: + - basename: medium-statefulset + objectTemplatePath: statefulset.yaml + - basename: medium-statefulset + objectTemplatePath: statefulset_service.yaml + {{end}} + +- name: Waiting for pods to be deleted + measurements: + - Identifier: WaitForRunningDeployments + Method: WaitForControlledPodsRunning + Params: + action: gather + {{if $ENABLE_STATEFULSETS}} + - Identifier: WaitForRunningStatefulSets + Method: WaitForControlledPodsRunning + Params: + action: gather + {{end}} + +- name: Deleting SVCs + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: Sequence + objectBundle: + - basename: big-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: Sequence + objectBundle: + - basename: medium-service + objectTemplatePath: service.yaml + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: Sequence + objectBundle: + - basename: small-service + objectTemplatePath: service.yaml + +- name: Collecting measurements + measurements: + - Identifier: APIResponsiveness + Method: APIResponsiveness + Params: + action: gather + - Identifier: APIResponsivenessPrometheus + Method: APIResponsivenessPrometheus + Params: + action: gather + {{if $ENABLE_PROMETHEUS_API_RESPONSIVENESS}} + enableViolations: true + {{end}} + - Identifier: PodStartupLatency + Method: PodStartupLatency + Params: + action: gather + - Identifier: InClusterNetworkLatency + Method: InClusterNetworkLatency + Params: + action: gather + - Identifier: DnsLookupLatency + Method: DnsLookupLatency + Params: + action: gather + - Identifier: NetworkProgrammingLatency + Method: NetworkProgrammingLatency + Params: + action: gather + - Identifier: TestMetrics + Method: TestMetrics + Params: + action: gather diff --git a/load_testing/k3s/cluster-loader/small/configmap.yaml b/load_testing/k3s/cluster-loader/small/configmap.yaml new file mode 100644 index 0000000000..b249a39143 --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.Name}} +data: + data.yaml: |- + a: 1 + b: 2 + c: 3 diff --git a/load_testing/k3s/cluster-loader/small/deployment.yaml b/load_testing/k3s/cluster-loader/small/deployment.yaml new file mode 100644 index 0000000000..dcd581914a --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/deployment.yaml @@ -0,0 +1,62 @@ +{{$EnableConfigMaps := DefaultParam .ENABLE_CONFIGMAPS false}} +{{$EnableSecrets := DefaultParam .ENABLE_SECRETS false}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.Name}} + labels: + group: load + svc: {{.SvcName}}-{{.Index}} +spec: + replicas: {{RandIntRange .ReplicasMin .ReplicasMax}} + selector: + matchLabels: + name: {{.Name}} + template: + metadata: + labels: + group: load + name: {{.Name}} + svc: {{.SvcName}}-{{.Index}} + spec: + containers: + - image: k8s.gcr.io/pause:3.1 + name: {{.Name}} + resources: + requests: + cpu: 10m + memory: "10M" + volumeMounts: + {{if and $EnableConfigMaps (eq (Mod .Index 20) 0 19) }} # .Index % 20 in {0,19} - 10% deployments will have ConfigMap + - name: configmap + mountPath: /var/configmap + {{end}} + {{if and $EnableSecrets (eq (Mod .Index 20) 10 19) }} # .Index % 20 in {10,19} - 10% deployments will have Secret + - name: secret + mountPath: /var/secret + {{end}} + dnsPolicy: Default + terminationGracePeriodSeconds: 1 + # Add not-ready/unreachable tolerations for 15 minutes so that node + # failure doesn't trigger pod deletion. + tolerations: + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 900 + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 900 + volumes: + {{if and $EnableConfigMaps (eq (Mod .Index 20) 0 19) }} # .Index % 20 in {0,19} - 10% deployments will have ConfigMap + - name: configmap + configMap: + name: {{.BaseName}}-{{.Index}} + {{end}} + {{if and $EnableSecrets (eq (Mod .Index 20) 10 19) }} # .Index % 20 in {10,19} - 10% deployments will have Secret + - name: secret + secret: + secretName: {{.BaseName}}-{{.Index}} + {{end}} diff --git a/load_testing/k3s/cluster-loader/small/run-test.sh b/load_testing/k3s/cluster-loader/small/run-test.sh new file mode 100644 index 0000000000..b190c99ef3 --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/run-test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +clusterloader --kubeconfig=../kubeConfig.yaml --testconfig=config.yaml diff --git a/load_testing/k3s/cluster-loader/small/secret.yaml b/load_testing/k3s/cluster-loader/small/secret.yaml new file mode 100644 index 0000000000..67134b355f --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} +type: Opaque +data: + password: c2NhbGFiaWxpdHkK diff --git a/load_testing/k3s/cluster-loader/small/service.yaml b/load_testing/k3s/cluster-loader/small/service.yaml new file mode 100644 index 0000000000..ed6a22c8cf --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/service.yaml @@ -0,0 +1,16 @@ +{{$SetServiceProxyLabel := DefaultParam .SetServiceProxyLabel false}} + +apiVersion: v1 +kind: Service +metadata: + name: {{.Name}} +{{if and $SetServiceProxyLabel (eq (Mod .Index 2) 0)}} + labels: + service.kubernetes.io/service-proxy-name: foo +{{end}} +spec: + selector: + svc: {{.Name}} + ports: + - port: 80 + targetPort: 80 diff --git a/load_testing/k3s/cluster-loader/small/statefulset.yaml b/load_testing/k3s/cluster-loader/small/statefulset.yaml new file mode 100644 index 0000000000..bb97bfce9a --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/statefulset.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{.Name}} + labels: + group: load +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + name: {{.Name}} + serviceName: {{.Name}} + replicas: {{RandIntRange .ReplicasMin .ReplicasMax}} + template: + metadata: + labels: + group: statefulset + name: {{.Name}} + spec: + terminationGracePeriodSeconds: 1 + containers: + - name: {{.Name}} + image: k8s.gcr.io/pause:3.1 + ports: + - containerPort: 80 + name: web + resources: + requests: + cpu: 10m + memory: "10M" diff --git a/load_testing/k3s/cluster-loader/small/statefulset_service.yaml b/load_testing/k3s/cluster-loader/small/statefulset_service.yaml new file mode 100644 index 0000000000..5e16a47a19 --- /dev/null +++ b/load_testing/k3s/cluster-loader/small/statefulset_service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.Name}} + labels: + name: {{.Name}} +spec: + clusterIP: None + selector: + name: {{.Name}} diff --git a/load_testing/k3s/pool/data.tf b/load_testing/k3s/pool/data.tf new file mode 100644 index 0000000000..0da33a9128 --- /dev/null +++ b/load_testing/k3s/pool/data.tf @@ -0,0 +1,55 @@ +data "terraform_remote_state" "server" { + backend = "local" + + config = { + path = "${path.module}/../server/server.tfstate" + } +} + +data "aws_vpc" "default" { + default = true +} + +data "aws_subnet_ids" "available" { + vpc_id = data.aws_vpc.default.id +} + +data "aws_subnet" "selected" { + id = "${tolist(data.aws_subnet_ids.available.ids)[1]}" +} + +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] + + filter { + name = "name" + values = ["ubuntu-minimal/images/*/ubuntu-bionic-18.04-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } +} + +data "template_file" "k3s-pool-worker-user_data" { + template = file("${path.module}/files/pool_worker_userdata.tmpl") + + vars = { + k3s_url = data.terraform_remote_state.server.outputs.public_ip[0] + k3s_cluster_secret = local.k3s_cluster_secret + install_k3s_version = local.install_k3s_version + k3s_per_node = var.k3s_per_node + } +} diff --git a/load_testing/k3s/pool/files/pool_worker_userdata.tmpl b/load_testing/k3s/pool/files/pool_worker_userdata.tmpl new file mode 100644 index 0000000000..c2d83f7343 --- /dev/null +++ b/load_testing/k3s/pool/files/pool_worker_userdata.tmpl @@ -0,0 +1,32 @@ +#cloud-config +ssh_authorized_keys: +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC06Qvs+Y9JiyOTeYNGAN/Ukq7SmeCTr7EreD1K8Lwu5VuOmo+SBZh685tNTEGV044HgFvGEOBVreDlO2ArYuwHjUBGnpQGV8/abjoeLrmZBdREAUzBQ1h2GFE/WssKUfum81cnigRK1J3tWP7emq/Y2h/Zw5F09yiCIlXMBX2auKWUCXqwG3xKTi1NVSF9N6BGyFolrAR0LZJ6k7UBXPRc/QDTclI427gSJNbnmn8LVym6YxacV/V9Y7s23iR5zYbhLPe9VJWYNk1brVvfUVb3mILVVYz76KGEq8SHdWlPQPCOp+fSJ+PezDRklnex/MmvhNrBOmMSNcpj7wSLA3hD wmaxwell@wmaxwell-laptop +- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2TBZGjE+J8ag11dzkFT58J3XPONDrVmalCNrKxsfADfyy0eqdZrG8hAcxAR/5zuj90Gin2uB4RSw6Cn4VHsPZcFpXyQCj1KQDADj+WcuhpXOIOY3AB0LZBly9NI0ll+8lo3QtEaoyRLtrMBhQ6Mooy2M3MTG4JNwU9o3yInuqZWf9PvtW6KxMl+ygg1xZkljhemGZ9k0wSrjqif+8usNbzVlCOVQmZwZA+BZxbdcLNwkg7zWJSXzDIXyqM6iWPGXQDEbWLq3+HR1qKucTCSxjbqoe0FD5xcW7NHIME5XKX84yH92n6yn+rxSsyUfhJWYqJd+i0fKf5UbN6qLrtd/D darren@darrens +runcmd: + - echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf + - echo "fs.file-max = 12000500" >> /etc/sysctl.conf + - echo "fs.nr_open = 20000500" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf + - echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf + - echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf + - ulimit -n 20000000 + - echo "# " >> /etc/security/limits.d/limits.conf + - echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf + - echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf + - sysctl -p + - apt-get update + - apt-get install -y software-properties-common + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - apt-get update + - apt-get -y install docker-ce + - apt-get install -y resolvconf linux-headers-$(uname -r) && echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail && systemctl start resolvconf + - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y + - n=1; while [ $n -le ${k3s_per_node} ]; do docker run -d --restart=unless-stopped -e K3S_URL=https://${k3s_url}:6443 -e K3S_CLUSTER_SECRET="${k3s_cluster_secret}" --privileged --mount type=tmpfs,destination=/var/run --mount type=tmpfs,destination=/run -m 1g --cpus=".7" rancher/k3s:${install_k3s_version}; n=$(( n + 1 )); done diff --git a/load_testing/k3s/pool/main.tf b/load_testing/k3s/pool/main.tf new file mode 100644 index 0000000000..6e85a15742 --- /dev/null +++ b/load_testing/k3s/pool/main.tf @@ -0,0 +1,80 @@ +terraform { + backend "local" { + path = "pool.tfstate" + } +} + +locals { + name = "load-test-pool" + k3s_cluster_secret = "pvc-6476dcaf-73a0-11e9-b8e5-06943b744282" + install_k3s_version = "v0.9.0-rc2" +} + +provider "aws" { + region = "us-west-2" + profile = "rancher-eng" +} + +resource "aws_security_group" "k3s" { + name = "${local.name}-pool" + vpc_id = data.aws_vpc.default.id + + ingress { + from_port = 22 + to_port = 22 + protocol = "TCP" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +module "k3s-pool-worker-asg" { + source = "terraform-aws-modules/autoscaling/aws" + version = "3.0.0" + name = local.name + asg_name = local.name + instance_type = var.worker_instance_type + image_id = data.aws_ami.ubuntu.id + user_data = data.template_file.k3s-pool-worker-user_data.rendered + ebs_optimized = true + + desired_capacity = var.node_count + health_check_type = "EC2" + max_size = var.node_count + min_size = var.node_count + vpc_zone_identifier = [data.aws_subnet.selected.id] + spot_price = "0.680" + + security_groups = [ + aws_security_group.k3s.id, + ] + + lc_name = local.name + + root_block_device = [ + { + volume_size = "100" + volume_type = "gp2" + }, + ] +} diff --git a/load_testing/k3s/pool/outputs.tf b/load_testing/k3s/pool/outputs.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/load_testing/k3s/pool/variables.tf b/load_testing/k3s/pool/variables.tf new file mode 100644 index 0000000000..0fef717b51 --- /dev/null +++ b/load_testing/k3s/pool/variables.tf @@ -0,0 +1,16 @@ +variable "node_count" { + description = "Number of nodes to run k3s agents on." + type = number + # default = 10 +} + +variable "k3s_per_node" { + description = "Number of k3s agent docker containers to run per ec2 instance" + type = number + default = 10 +} + +variable "worker_instance_type" { + type = string + default = "c5.4xlarge" +} diff --git a/load_testing/k3s/pool/versions.tf b/load_testing/k3s/pool/versions.tf new file mode 100644 index 0000000000..ac97c6ac8e --- /dev/null +++ b/load_testing/k3s/pool/versions.tf @@ -0,0 +1,4 @@ + +terraform { + required_version = ">= 0.12" +} diff --git a/load_testing/k3s/readme.MD b/load_testing/k3s/readme.MD new file mode 100644 index 0000000000..3b3d1154ea --- /dev/null +++ b/load_testing/k3s/readme.MD @@ -0,0 +1,47 @@ +# K3S Load Testing + +This directory contains tooling to help spin up k3s clusters for scale testing (load testing the k3s server). + +## Usage + +From inside the `server` directory, run the following commands: + +``` +cd server +terraform init +terraform apply +``` + +You will be asked to specify an instance type for the k3s server. For a `large` cluster test, use a `c4.8xlarge`. For a `small` cluster test, use a `t3.micro`. +To run these commands, you will need access to an AWS account (configured here by default will be the `rancher-eng` account). + +When the server terraform completes, go to the `pool` directory and run: + +``` +cd pool +terraform init +terraform apply +``` + +You will be asked to specify how many ec2 instances to create (variable is `node_count`). You can also specify the `k3s_per_node` and `worker_instance_type` variables when you run apply to override these defaults. + +For the `large` cluster test, you will want to specify `node_count=100`. That will get you 100 ec2 instances with 10 k3s agents each - for a total of 1000 nodes in your k3s cluster. + +For the `small` test, `node_count=1`, and you will override `k3s_per_node=5`. That will get you 1 ec2 instance with 5 agents on it - for a total of 5 nodes in your k3s cluster. + + +Once `pool` is finished, you can run through the cluster-loader scenarios using the `run-test.sh` script in the corresponding directory (depending on whether you are running the `large` or `small` scenario). + +``` +cd cluster-loader/ +./run-test.sh +``` + +* The `run-test.sh` script assumes you have [cluster-loader](https://github.com/kubernetes/perf-tests/tree/master/clusterloader2) installed on your machine. + + +### TODO + +* Investigate cluster-loader failures. +* Simplify this process. +* Organized reporting on SLOs after cluster-loader is complete. diff --git a/load_testing/k3s/server/data.tf b/load_testing/k3s/server/data.tf new file mode 100644 index 0000000000..2a8bb32025 --- /dev/null +++ b/load_testing/k3s/server/data.tf @@ -0,0 +1,83 @@ +data "aws_vpc" "default" { + default = true +} + +data "aws_subnet_ids" "available" { + vpc_id = data.aws_vpc.default.id +} + +data "aws_subnet" "selected" { + id = "${tolist(data.aws_subnet_ids.available.ids)[1]}" +} + +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] + + filter { + name = "name" + values = ["ubuntu-minimal/images/*/ubuntu-bionic-18.04-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } +} + +data "template_file" "metrics" { + template = file("${path.module}/files/metrics.yaml") +} +data "template_file" "k3s-prom-yaml" { + template = file("${path.module}/files/prom.yaml") + vars = { + prom_host = var.prom_host + graf_host = var.graf_host + } +} + +data "template_file" "k3s-server-user_data" { + template = file("${path.module}/files/server_userdata.tmpl") + + vars = { + create_eip = 1 + metrics_yaml = base64encode(data.template_file.metrics.rendered) + prom_yaml = base64encode(data.template_file.k3s-prom-yaml.rendered) + eip = join(",", aws_eip.k3s-server.*.public_ip) + k3s_cluster_secret = local.k3s_cluster_secret + install_k3s_version = local.install_k3s_version + k3s_server_args = var.k3s_server_args + } +} + +data "template_file" "k3s-prom-worker-user_data" { + template = file("${path.module}/files/worker_userdata.tmpl") + + vars = { + k3s_url = aws_eip.k3s-server.0.public_ip + k3s_cluster_secret = local.k3s_cluster_secret + install_k3s_version = local.install_k3s_version + k3s_exec = "--node-label prom=true" + } +} + +data "template_file" "k3s-worker-user_data" { + template = file("${path.module}/files/worker_userdata.tmpl") + + vars = { + k3s_url = aws_eip.k3s-server.0.public_ip + k3s_cluster_secret = local.k3s_cluster_secret + install_k3s_version = local.install_k3s_version + k3s_exec = "" + } +} diff --git a/load_testing/k3s/server/files/metrics.yaml b/load_testing/k3s/server/files/metrics.yaml new file mode 100644 index 0000000000..3b35b737d6 --- /dev/null +++ b/load_testing/k3s/server/files/metrics.yaml @@ -0,0 +1,227 @@ +%{ if local.prom_worker_node_count != 0 } +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kube-state-metrics +rules: +- apiGroups: [""] + resources: + - configmaps + - secrets + - nodes + - pods + - services + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: ["list", "watch"] +- apiGroups: ["extensions"] + resources: + - daemonsets + - deployments + - replicasets + - ingresses + verbs: ["list", "watch"] +- apiGroups: ["apps"] + resources: + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["list", "watch"] +- apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: ["list", "watch"] +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +- apiGroups: ["autoscaling.k8s.io"] + resources: + - verticalpodautoscalers + verbs: ["list", "watch"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: kube-state-metrics + name: kube-state-metrics + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-state-metrics + replicas: 1 + template: + metadata: + labels: + k8s-app: kube-state-metrics + spec: + serviceAccountName: kube-state-metrics + containers: + - name: kube-state-metrics + image: quay.io/coreos/kube-state-metrics:v1.7.2 + ports: + - name: http-metrics + containerPort: 8080 + - name: telemetry + containerPort: 8081 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-state-metrics + namespace: kube-system + labels: + k8s-app: kube-state-metrics + annotations: + prometheus.io/scrape: 'true' +spec: + ports: + - name: http-metrics + port: 8080 + targetPort: http-metrics + protocol: TCP + - name: telemetry + port: 8081 + targetPort: telemetry + protocol: TCP + selector: + k8s-app: kube-state-metrics +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: slo-monitor +subjects: +- kind: ServiceAccount + name: slo-monitor + namespace: kube-system +roleRef: + kind: ClusterRole + name: slo-monitor + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: slo-monitor + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["pods", "events"] + verbs: ["get", "watch", "list"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: slo-monitor + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: slo-monitor + namespace: kube-system + labels: + app: slo-monitor +spec: + selector: + matchLabels: + app: slo-monitor + template: + metadata: + labels: + app: slo-monitor + annotations: + prometheus.io/scrape: "true" + spec: + containers: + - name: slo-monitor + image: gcr.io/google-containers/slo-monitor:0.12.0 + command: + - /slo-monitor + - --alsologtostderr=true + imagePullPolicy: Always + ports: + - name: metrics + containerPort: 8080 + resources: + requests: + cpu: 300m + memory: 100Mi + limits: + cpu: 300m + memory: 100Mi + restartPolicy: Always + serviceAccountName: slo-monitor +--- +apiVersion: v1 +kind: Service +metadata: + name: slo-monitor + namespace: kube-system + labels: + app: slo-monitor +spec: + selector: + app: slo-monitor + ports: + - name: metrics + port: 80 + targetPort: metrics + type: LoadBalancer +%{ endif } diff --git a/load_testing/k3s/server/files/prom.yaml b/load_testing/k3s/server/files/prom.yaml new file mode 100644 index 0000000000..9c780b2d36 --- /dev/null +++ b/load_testing/k3s/server/files/prom.yaml @@ -0,0 +1,86 @@ +%{ if local.prom_worker_node_count != 0 } +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: prometheus + namespace: kube-system +spec: + chart: https://raw.githubusercontent.com/drpebcak/charts/master/prometheus-9.1.0.tgz + targetNamespace: monitoring + valuesContent: |- + alertmanager: + nodeSelector: + prom: "true" + persistentVolume: + enabled: false + kubeStateMetrics: + nodeSelector: + prom: "true" + nodeExporter: + nodeSelector: + prom: "true" + server: + nodeSelector: + prom: "true" + ingress: + enabled: true + hosts: + - ${prom_host} + persistentVolume: + enabled: false + pushgateway: + nodeSelector: + prom: "true" + persistentVolume: + enabled: false + serverFiles: + prometheus.yml: + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - job_name: kubernetes-apiservers + scrape_interval: 10s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + kubernetes_sd_configs: + - api_server: null + role: endpoints + namespaces: + names: [] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + separator: ; + regex: default;kubernetes;https + replacement: $1 + action: keep +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: grafana + namespace: kube-system +spec: + chart: stable/grafana + targetNamespace: monitoring + valuesContent: |- + ingress: + enabled: true + hosts: + - ${graf_host} + nodeSelector: + prom: "true" +%{ endif } diff --git a/load_testing/k3s/server/files/server_userdata.tmpl b/load_testing/k3s/server/files/server_userdata.tmpl new file mode 100644 index 0000000000..74e1cbf41e --- /dev/null +++ b/load_testing/k3s/server/files/server_userdata.tmpl @@ -0,0 +1,45 @@ +#cloud-config +ssh_authorized_keys: +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC06Qvs+Y9JiyOTeYNGAN/Ukq7SmeCTr7EreD1K8Lwu5VuOmo+SBZh685tNTEGV044HgFvGEOBVreDlO2ArYuwHjUBGnpQGV8/abjoeLrmZBdREAUzBQ1h2GFE/WssKUfum81cnigRK1J3tWP7emq/Y2h/Zw5F09yiCIlXMBX2auKWUCXqwG3xKTi1NVSF9N6BGyFolrAR0LZJ6k7UBXPRc/QDTclI427gSJNbnmn8LVym6YxacV/V9Y7s23iR5zYbhLPe9VJWYNk1brVvfUVb3mILVVYz76KGEq8SHdWlPQPCOp+fSJ+PezDRklnex/MmvhNrBOmMSNcpj7wSLA3hD wmaxwell@wmaxwell-laptop +- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2TBZGjE+J8ag11dzkFT58J3XPONDrVmalCNrKxsfADfyy0eqdZrG8hAcxAR/5zuj90Gin2uB4RSw6Cn4VHsPZcFpXyQCj1KQDADj+WcuhpXOIOY3AB0LZBly9NI0ll+8lo3QtEaoyRLtrMBhQ6Mooy2M3MTG4JNwU9o3yInuqZWf9PvtW6KxMl+ygg1xZkljhemGZ9k0wSrjqif+8usNbzVlCOVQmZwZA+BZxbdcLNwkg7zWJSXzDIXyqM6iWPGXQDEbWLq3+HR1qKucTCSxjbqoe0FD5xcW7NHIME5XKX84yH92n6yn+rxSsyUfhJWYqJd+i0fKf5UbN6qLrtd/D darren@darrens +write_files: + - path: /var/lib/rancher/k3s/server/manifests/metrics.yaml + permissions: "0755" + owner: root:root + encoding: b64 + content: ${metrics_yaml} + - path: /var/lib/rancher/k3s/server/manifests/prom.yaml + permissions: "0755" + owner: root:root + encoding: b64 + content: ${prom_yaml} +runcmd: + - echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf + - echo "fs.file-max = 12000500" >> /etc/sysctl.conf + - echo "fs.nr_open = 20000500" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf + - echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf + - echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf + - ulimit -n 20000000 + - echo "# " >> /etc/security/limits.d/limits.conf + - echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf + - echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf + - sysctl -p + - apt-get update + - apt-get install -y software-properties-common resolvconf linux-headers-$(uname -r) + - echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail + - systemctl start resolvconf + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - apt-get update + - apt-get -y install docker-ce + - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y + - if [ "${create_eip}" = "1" ]; then docker run -e "EIP=${eip}" cloudnautique/eip-autoassign:latest; fi + - until (curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="${k3s_server_args} --disable-agent --no-deploy traefik --no-deploy servicelb --cluster-cidr=10.0.0.0/8 --service-cidr=192.168.0.0/16 --cluster-dns=192.168.0.10 --tls-san ${eip}" K3S_CLUSTER_SECRET="${k3s_cluster_secret}" INSTALL_K3S_VERSION=${install_k3s_version} sh -); do echo 'Error installing k3s'; sleep 1; done diff --git a/load_testing/k3s/server/files/worker_userdata.tmpl b/load_testing/k3s/server/files/worker_userdata.tmpl new file mode 100644 index 0000000000..8d5e7a55fd --- /dev/null +++ b/load_testing/k3s/server/files/worker_userdata.tmpl @@ -0,0 +1,26 @@ +#cloud-config +ssh_authorized_keys: +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC06Qvs+Y9JiyOTeYNGAN/Ukq7SmeCTr7EreD1K8Lwu5VuOmo+SBZh685tNTEGV044HgFvGEOBVreDlO2ArYuwHjUBGnpQGV8/abjoeLrmZBdREAUzBQ1h2GFE/WssKUfum81cnigRK1J3tWP7emq/Y2h/Zw5F09yiCIlXMBX2auKWUCXqwG3xKTi1NVSF9N6BGyFolrAR0LZJ6k7UBXPRc/QDTclI427gSJNbnmn8LVym6YxacV/V9Y7s23iR5zYbhLPe9VJWYNk1brVvfUVb3mILVVYz76KGEq8SHdWlPQPCOp+fSJ+PezDRklnex/MmvhNrBOmMSNcpj7wSLA3hD wmaxwell@wmaxwell-laptop +- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak +runcmd: + - echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf + - echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf + - echo "fs.file-max = 12000500" >> /etc/sysctl.conf + - echo "fs.nr_open = 20000500" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf + - echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf + - echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf + - echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf + - ulimit -n 20000000 + - echo "# " >> /etc/security/limits.d/limits.conf + - echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf + - echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf + - sysctl -p + - apt-get update + - apt-get install -y software-properties-common + - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y + - until (curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${install_k3s_version} INSTALL_K3S_EXEC="${k3s_exec}" K3S_URL=https://${k3s_url}:6443 K3S_CLUSTER_SECRET="${k3s_cluster_secret}" sh -); do echo 'k3s did not install correctly'; sleep 1; done diff --git a/load_testing/k3s/server/iam.tf b/load_testing/k3s/server/iam.tf new file mode 100644 index 0000000000..3b7c786860 --- /dev/null +++ b/load_testing/k3s/server/iam.tf @@ -0,0 +1,58 @@ +resource "aws_iam_instance_profile" "k3s-server" { + name_prefix = "load-testing-k3s-server" + role = aws_iam_role.k3s-server.name + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_iam_role" "k3s-server" { + name_prefix = "load-testing-k3s-server" + + assume_role_policy = <| ../cluster-loader/kubeConfig.yaml; do sleep 5; done" + } +} diff --git a/load_testing/k3s/server/outputs.tf b/load_testing/k3s/server/outputs.tf new file mode 100644 index 0000000000..a994aafb68 --- /dev/null +++ b/load_testing/k3s/server/outputs.tf @@ -0,0 +1,11 @@ +output "public_ip" { + value = aws_eip.k3s-server.*.public_ip +} + +output "install_k3s_version" { + value = local.install_k3s_version +} + +output "k3s_cluster_secret" { + value = local.k3s_cluster_secret +} diff --git a/load_testing/k3s/server/variables.tf b/load_testing/k3s/server/variables.tf new file mode 100644 index 0000000000..779ac4ffcb --- /dev/null +++ b/load_testing/k3s/server/variables.tf @@ -0,0 +1,12 @@ +variable "server_instance_type" { + # default = "c4.8xlarge" +} +variable "k3s_server_args" { + default = "" +} +variable "prom_host" { + default = "" +} +variable "graf_host" { + default = "" +} diff --git a/load_testing/k3s/server/versions.tf b/load_testing/k3s/server/versions.tf new file mode 100644 index 0000000000..ac97c6ac8e --- /dev/null +++ b/load_testing/k3s/server/versions.tf @@ -0,0 +1,4 @@ + +terraform { + required_version = ">= 0.12" +}