add dqlite

tests/perf/agents/files/pool_worker_userdata.tmpl

@@ -27,7 +27,4 @@ runcmd:
 - apt-get install -y software-properties-common
 - apt-get install -y resolvconf linux-headers-$(uname -r) && echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail && systemctl start resolvconf
 - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
-- wget https://raw.githubusercontent.com/galal-hussein/k3s/k3s_with_kine_fix/k3s
-- cp k3s /usr/local/bin/k3s
-- chmod +x /usr/local/bin/k3s
 - until (curl -sfL https://get.k3s.io | K3S_URL=https://${k3s_url}:6443 K3S_CLUSTER_SECRET="${k3s_cluster_secret}" K3S_CLUSTER_SECRET="${k3s_cluster_secret}" INSTALL_K3S_VERSION="${install_k3s_version}" sh -); do echo 'Error installing k3s agent'; sleep 1; done

tests/perf/agents/main.tf

@@ -10,7 +10,7 @@ locals {
 }
 
 provider "aws" {
-  region  = "us-west-2"
+  region  = "us-east-2"
   profile = "rancher-eng"
 }
 
@@ -76,7 +76,7 @@ module "k3s-pool-worker-asg" {
 
   root_block_device = [
     {
-      volume_size = "100"
+      volume_size = "30"
       volume_type = "gp2"
     },
   ]

tests/perf/scripts/config

@@ -1,21 +1,21 @@
 ## MAIN VARIABLES ##
 ####################
 CLUSTER_NAME="loadtest-k3s"
-DOMAIN_NAME=""
+DOMAIN_NAME="loadtest.eng.rancher.space"
 ZONE_ID=""
-K3S_VERSION="v0.11.0-alpha2"
+K3S_VERSION="v1.0.0"
 EXTRA_SSH_KEYS="" # comma separated public keys
 PRIVATE_KEY_PATH="~/.ssh/id_rsa"
 DEBUG=1
 
 ## K3S DB VARIABLES ##
 ##########################
-DB_ENGINE="postgres"
+DB_ENGINE="dqlite"
 DB_INSTANCE_TYPE="db.m4.4xlarge"
 DB_NAME="k3s"
 DB_USERNAME="k3suser"
 DB_PASSWORD="024d9442b3add64b7ef90655bc302cd8"
-DB_VERSION=11.5
+DB_VERSION=5.7
 
 ## K3S SERVER VARIABLES ##
 ##########################
@@ -30,5 +30,5 @@ PROM_WORKER_INSTANCE_TYPE="m5.large"
 
 ## K3S AGENTS VARIABLES ##
 ##########################
-AGENT_NODE_COUNT=100
+AGENT_NODE_COUNT=10
 AGENT_INSTANCE_TYPE="m5.large"

tests/perf/server/files/server_userdata.tmpl

@@ -16,18 +16,24 @@ write_files:
       STORAGE_ENDPOINT="postgres://${db_username}:${db_password}@${db_address}:5432/${db_name}"
     elif [ ${db_engine} == "mysql" ]; then
       STORAGE_ENDPOINT="mysql://${db_username}:${db_password}@(${db_address})/${db_name}"
-    else
+    elif [ ${db_engine} == "etcd" ]; then
      IFS=',' read -r -a private_ips <<< "${db_address}"
      for i in "$${!private_ips[@]}"; do
       STORAGE_ENDPOINT=$STORAGE_ENDPOINT"http://$${private_ips[i]}:2379",
      done
      STORAGE_ENDPOINT=$${STORAGE_ENDPOINT%?}
-     echo hello
     fi
+
     while true; do
+    if [ ${db_engine} == "dqlite" ]; then
+    curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \
+       INSTALL_K3S_VERSION="${install_k3s_version}" \
+       INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if master_index != 0 } --server https://${lb_address}:6443 %{ else } --cluster-init %{ endif }" sh -
+    else
       curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \
          INSTALL_K3S_VERSION="${install_k3s_version}" \
-         INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --storage-endpoint=$STORAGE_ENDPOINT %{ endif }" sh -
+         INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --datastore-endpoint=$STORAGE_ENDPOINT %{ endif }" sh -
+      fi
       if [ $? -eq 0 ]; then
         break
       fi

tests/perf/server/files/worker_userdata.tmpl

@@ -23,7 +23,4 @@ runcmd:
 - echo "    *       soft  nofile  20000" >> /etc/security/limits.d/limits.conf
 - echo "    *       hard  nofile  20000" >> /etc/security/limits.d/limits.conf
 - sysctl -p
-- wget https://raw.githubusercontent.com/galal-hussein/k3s/scale_test/k3s
-- cp k3s /usr/local/bin/k3s
-- chmod +x /usr/local/bin/k3s
 - until (curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${install_k3s_version} INSTALL_K3S_EXEC="${k3s_exec}" K3S_URL=https://${k3s_url}:6443 K3S_CLUSTER_SECRET="${k3s_cluster_secret}" sh -); do echo 'k3s did not install correctly'; sleep 1; done

tests/perf/server/main.tf

@@ -13,7 +13,7 @@ locals {
 }
 
 provider "aws" {
-  region  = "us-west-2"
+  region  = "us-east-2"
   profile = "rancher-eng"
 }
 
@@ -58,10 +58,9 @@ resource "aws_security_group" "k3s" {
 }
 
 resource "aws_db_instance" "k3s_db" {
-  count                = "${var.db_engine == "etcd" ? 0 : var.server_ha}"
+  count                = "${var.db_engine == "postgres" || var.db_engine == "mysql" ? 1 : 0 }"
   allocated_storage    = 100 #baseline iops is 300 with gp2
-  storage_type         = "io1"
+  storage_type         = "gp2"
-  iops                 = "3000"
   engine               = "${var.db_engine}"
   engine_version       = "${var.db_version}"
   instance_class       = "${var.db_instance_type}"
@@ -87,7 +86,7 @@ resource "aws_instance" "k3s_etcd" {
   ]
 
    root_block_device {
-    volume_size = "100"
+    volume_size = "30"
     volume_type = "gp2"
   }
 
@@ -156,8 +155,8 @@ resource "aws_instance" "k3s-server" {
     k3s_cluster_secret = local.k3s_cluster_secret,
     install_k3s_version = local.install_k3s_version,
     k3s_server_args = var.k3s_server_args,
-    db_engine = var.db_engine
+    db_engine = var.db_engine,
-    db_address = "${var.db_engine == "etcd" ? join(",",aws_instance.k3s_etcd.*.private_ip) : aws_db_instance.k3s_db[0].address}",
+    db_address = "${var.db_engine == "etcd" ? join(",",aws_instance.k3s_etcd.*.private_ip) : var.db_engine == "dqlite" ? "null" : aws_db_instance.k3s_db[0].address}",
     db_name = var.db_name,
     db_username = var.db_username,
     db_password = var.db_password,
@@ -172,12 +171,14 @@ resource "aws_instance" "k3s-server" {
   ]
 
    root_block_device {
-    volume_size = "100"
+    volume_size = "30"
     volume_type = "gp2"
   }
 
    tags = {
     Name = "${local.name}-server-${count.index}"
+    Role = "master"
+    Leader = "${count.index == 0 ? "true" : "false"}"
   }
   provisioner "local-exec" {
       command = "sleep 10"
@@ -208,7 +209,7 @@ module "k3s-prom-worker-asg" {
 
   root_block_device = [
     {
-      volume_size = "100"
+      volume_size = "30"
       volume_type = "gp2"
     },
   ]

tests/perf/tests/load/config.yaml

@@ -6,11 +6,11 @@
 
 #Constants
 {{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}}
-{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 100}}
+{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 10}}
-{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 100}}
+{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 30}}
 {{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}}
-{{$BIG_GROUP_SIZE := 1000}}
+{{$BIG_GROUP_SIZE := 300}}
-{{$MEDIUM_GROUP_SIZE := 500}}
+{{$MEDIUM_GROUP_SIZE := 150}}
 {{$SMALL_GROUP_SIZE := 50}}
 {{$SMALL_STATEFUL_SETS_PER_NAMESPACE := 1}}
 {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE := 1}}