From 9969dc9cff92b65dc0ac5a37c4921cbf18d01222 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Fri, 15 Nov 2019 13:14:28 +0200 Subject: [PATCH] add dqlite --- .../agents/files/pool_worker_userdata.tmpl | 3 --- tests/perf/agents/main.tf | 4 ++-- tests/perf/scripts/config | 10 +++++----- tests/perf/server/files/server_userdata.tmpl | 12 +++++++++--- tests/perf/server/files/worker_userdata.tmpl | 3 --- tests/perf/server/main.tf | 19 ++++++++++--------- tests/perf/tests/load/config.yaml | 8 ++++---- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/tests/perf/agents/files/pool_worker_userdata.tmpl b/tests/perf/agents/files/pool_worker_userdata.tmpl index 6e08a5d300..b117a5635b 100644 --- a/tests/perf/agents/files/pool_worker_userdata.tmpl +++ b/tests/perf/agents/files/pool_worker_userdata.tmpl @@ -27,7 +27,4 @@ runcmd: - apt-get install -y software-properties-common - apt-get install -y resolvconf linux-headers-$(uname -r) && echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail && systemctl start resolvconf - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -- wget https://raw.githubusercontent.com/galal-hussein/k3s/k3s_with_kine_fix/k3s -- cp k3s /usr/local/bin/k3s -- chmod +x /usr/local/bin/k3s - until (curl -sfL https://get.k3s.io | K3S_URL=https://${k3s_url}:6443 K3S_CLUSTER_SECRET="${k3s_cluster_secret}" K3S_CLUSTER_SECRET="${k3s_cluster_secret}" INSTALL_K3S_VERSION="${install_k3s_version}" sh -); do echo 'Error installing k3s agent'; sleep 1; done diff --git a/tests/perf/agents/main.tf b/tests/perf/agents/main.tf index f62c432fd3..106b7e8dcb 100644 --- a/tests/perf/agents/main.tf +++ b/tests/perf/agents/main.tf @@ -10,7 +10,7 @@ locals { } provider "aws" { - region = "us-west-2" + region = "us-east-2" profile = "rancher-eng" } @@ -76,7 +76,7 @@ module "k3s-pool-worker-asg" { root_block_device = [ { - volume_size = "100" + volume_size = "30" volume_type = "gp2" }, ] diff --git a/tests/perf/scripts/config b/tests/perf/scripts/config index 3505846732..d85bf41426 100755 --- a/tests/perf/scripts/config +++ b/tests/perf/scripts/config @@ -1,21 +1,21 @@ ## MAIN VARIABLES ## #################### CLUSTER_NAME="loadtest-k3s" -DOMAIN_NAME="" +DOMAIN_NAME="loadtest.eng.rancher.space" ZONE_ID="" -K3S_VERSION="v0.11.0-alpha2" +K3S_VERSION="v1.0.0" EXTRA_SSH_KEYS="" # comma separated public keys PRIVATE_KEY_PATH="~/.ssh/id_rsa" DEBUG=1 ## K3S DB VARIABLES ## ########################## -DB_ENGINE="postgres" +DB_ENGINE="dqlite" DB_INSTANCE_TYPE="db.m4.4xlarge" DB_NAME="k3s" DB_USERNAME="k3suser" DB_PASSWORD="024d9442b3add64b7ef90655bc302cd8" -DB_VERSION=11.5 +DB_VERSION=5.7 ## K3S SERVER VARIABLES ## ########################## @@ -30,5 +30,5 @@ PROM_WORKER_INSTANCE_TYPE="m5.large" ## K3S AGENTS VARIABLES ## ########################## -AGENT_NODE_COUNT=100 +AGENT_NODE_COUNT=10 AGENT_INSTANCE_TYPE="m5.large" diff --git a/tests/perf/server/files/server_userdata.tmpl b/tests/perf/server/files/server_userdata.tmpl index 65145c7c68..e831429736 100644 --- a/tests/perf/server/files/server_userdata.tmpl +++ b/tests/perf/server/files/server_userdata.tmpl @@ -16,18 +16,24 @@ write_files: STORAGE_ENDPOINT="postgres://${db_username}:${db_password}@${db_address}:5432/${db_name}" elif [ ${db_engine} == "mysql" ]; then STORAGE_ENDPOINT="mysql://${db_username}:${db_password}@(${db_address})/${db_name}" - else + elif [ ${db_engine} == "etcd" ]; then IFS=',' read -r -a private_ips <<< "${db_address}" for i in "$${!private_ips[@]}"; do STORAGE_ENDPOINT=$STORAGE_ENDPOINT"http://$${private_ips[i]}:2379", done STORAGE_ENDPOINT=$${STORAGE_ENDPOINT%?} - echo hello fi + while true; do + if [ ${db_engine} == "dqlite" ]; then + curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \ + INSTALL_K3S_VERSION="${install_k3s_version}" \ + INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if master_index != 0 } --server https://${lb_address}:6443 %{ else } --cluster-init %{ endif }" sh - + else curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \ INSTALL_K3S_VERSION="${install_k3s_version}" \ - INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --storage-endpoint=$STORAGE_ENDPOINT %{ endif }" sh - + INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --datastore-endpoint=$STORAGE_ENDPOINT %{ endif }" sh - + fi if [ $? -eq 0 ]; then break fi diff --git a/tests/perf/server/files/worker_userdata.tmpl b/tests/perf/server/files/worker_userdata.tmpl index ae2aaa9609..e451a6d0ce 100644 --- a/tests/perf/server/files/worker_userdata.tmpl +++ b/tests/perf/server/files/worker_userdata.tmpl @@ -23,7 +23,4 @@ runcmd: - echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf - echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf - sysctl -p -- wget https://raw.githubusercontent.com/galal-hussein/k3s/scale_test/k3s -- cp k3s /usr/local/bin/k3s -- chmod +x /usr/local/bin/k3s - until (curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${install_k3s_version} INSTALL_K3S_EXEC="${k3s_exec}" K3S_URL=https://${k3s_url}:6443 K3S_CLUSTER_SECRET="${k3s_cluster_secret}" sh -); do echo 'k3s did not install correctly'; sleep 1; done diff --git a/tests/perf/server/main.tf b/tests/perf/server/main.tf index 9c5bdcceda..0e5e1895c8 100644 --- a/tests/perf/server/main.tf +++ b/tests/perf/server/main.tf @@ -13,7 +13,7 @@ locals { } provider "aws" { - region = "us-west-2" + region = "us-east-2" profile = "rancher-eng" } @@ -58,10 +58,9 @@ resource "aws_security_group" "k3s" { } resource "aws_db_instance" "k3s_db" { - count = "${var.db_engine == "etcd" ? 0 : var.server_ha}" + count = "${var.db_engine == "postgres" || var.db_engine == "mysql" ? 1 : 0 }" allocated_storage = 100 #baseline iops is 300 with gp2 - storage_type = "io1" - iops = "3000" + storage_type = "gp2" engine = "${var.db_engine}" engine_version = "${var.db_version}" instance_class = "${var.db_instance_type}" @@ -87,7 +86,7 @@ resource "aws_instance" "k3s_etcd" { ] root_block_device { - volume_size = "100" + volume_size = "30" volume_type = "gp2" } @@ -156,8 +155,8 @@ resource "aws_instance" "k3s-server" { k3s_cluster_secret = local.k3s_cluster_secret, install_k3s_version = local.install_k3s_version, k3s_server_args = var.k3s_server_args, - db_engine = var.db_engine - db_address = "${var.db_engine == "etcd" ? join(",",aws_instance.k3s_etcd.*.private_ip) : aws_db_instance.k3s_db[0].address}", + db_engine = var.db_engine, + db_address = "${var.db_engine == "etcd" ? join(",",aws_instance.k3s_etcd.*.private_ip) : var.db_engine == "dqlite" ? "null" : aws_db_instance.k3s_db[0].address}", db_name = var.db_name, db_username = var.db_username, db_password = var.db_password, @@ -172,12 +171,14 @@ resource "aws_instance" "k3s-server" { ] root_block_device { - volume_size = "100" + volume_size = "30" volume_type = "gp2" } tags = { Name = "${local.name}-server-${count.index}" + Role = "master" + Leader = "${count.index == 0 ? "true" : "false"}" } provisioner "local-exec" { command = "sleep 10" @@ -208,7 +209,7 @@ module "k3s-prom-worker-asg" { root_block_device = [ { - volume_size = "100" + volume_size = "30" volume_type = "gp2" }, ] diff --git a/tests/perf/tests/load/config.yaml b/tests/perf/tests/load/config.yaml index a0612bc371..a70765330a 100644 --- a/tests/perf/tests/load/config.yaml +++ b/tests/perf/tests/load/config.yaml @@ -6,11 +6,11 @@ #Constants {{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}} -{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 100}} -{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 100}} +{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 10}} +{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 30}} {{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}} -{{$BIG_GROUP_SIZE := 1000}} -{{$MEDIUM_GROUP_SIZE := 500}} +{{$BIG_GROUP_SIZE := 300}} +{{$MEDIUM_GROUP_SIZE := 150}} {{$SMALL_GROUP_SIZE := 50}} {{$SMALL_STATEFUL_SETS_PER_NAMESPACE := 1}} {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE := 1}}