Mirror/k3s
1
0
Fork 0
mirror of https://github.com/k3s-io/k3s.git synced 2024-06-07 19:41:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

general improvements, dont require iam

Browse Source
This commit is contained in:
Taylor Price 2019-10-17 16:21:42 -07:00 committed by galal-hussein
parent 677cc1a191
commit bf6ca8b2d8
11 changed files with 153 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
load_testing/k3s/pool/data.tf
View File

@ -42,14 +42,3 @@ data "aws_ami" "ubuntu" {
values = ["x86_64"] values = ["x86_64"]
} }
} }
data "template_file" "k3s-pool-worker-user_data" {
template = file("${path.module}/files/pool_worker_userdata.tmpl")
vars = {
k3s_url = data.terraform_remote_state.server.outputs.public_ip[0]
k3s_cluster_secret = local.k3s_cluster_secret
install_k3s_version = local.install_k3s_version
k3s_per_node = var.k3s_per_node
}
}

60
load_testing/k3s/pool/files/pool_worker_userdata.tmpl
View File

@ -1,32 +1,34 @@
#cloud-config #cloud-config
%{ if length(extra_ssh_keys) > 0 }
ssh_authorized_keys: ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC06Qvs+Y9JiyOTeYNGAN/Ukq7SmeCTr7EreD1K8Lwu5VuOmo+SBZh685tNTEGV044HgFvGEOBVreDlO2ArYuwHjUBGnpQGV8/abjoeLrmZBdREAUzBQ1h2GFE/WssKUfum81cnigRK1J3tWP7emq/Y2h/Zw5F09yiCIlXMBX2auKWUCXqwG3xKTi1NVSF9N6BGyFolrAR0LZJ6k7UBXPRc/QDTclI427gSJNbnmn8LVym6YxacV/V9Y7s23iR5zYbhLPe9VJWYNk1brVvfUVb3mILVVYz76KGEq8SHdWlPQPCOp+fSJ+PezDRklnex/MmvhNrBOmMSNcpj7wSLA3hD wmaxwell@wmaxwell-laptop %{ for ssh_key in extra_ssh_keys }
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak - ${ssh_key}
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2TBZGjE+J8ag11dzkFT58J3XPONDrVmalCNrKxsfADfyy0eqdZrG8hAcxAR/5zuj90Gin2uB4RSw6Cn4VHsPZcFpXyQCj1KQDADj+WcuhpXOIOY3AB0LZBly9NI0ll+8lo3QtEaoyRLtrMBhQ6Mooy2M3MTG4JNwU9o3yInuqZWf9PvtW6KxMl+ygg1xZkljhemGZ9k0wSrjqif+8usNbzVlCOVQmZwZA+BZxbdcLNwkg7zWJSXzDIXyqM6iWPGXQDEbWLq3+HR1qKucTCSxjbqoe0FD5xcW7NHIME5XKX84yH92n6yn+rxSsyUfhJWYqJd+i0fKf5UbN6qLrtd/D darren@darrens %{ endfor }
%{ endif }
runcmd: runcmd:
- echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf
- echo "fs.file-max = 12000500" >> /etc/sysctl.conf - echo "fs.file-max = 12000500" >> /etc/sysctl.conf
- echo "fs.nr_open = 20000500" >> /etc/sysctl.conf - echo "fs.nr_open = 20000500" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf
- echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf - echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf
- echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf - echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf
- ulimit -n 20000000 - ulimit -n 20000000
- echo "# <domain> <type> <item> <value>" >> /etc/security/limits.d/limits.conf - echo "# <domain> <type> <item> <value>" >> /etc/security/limits.d/limits.conf
- echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf - echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf
- echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf - echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf
- sysctl -p - sysctl -p
- apt-get update - apt-get update
- apt-get install -y software-properties-common - apt-get install -y software-properties-common
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
- add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- apt-get update - apt-get update
- apt-get -y install docker-ce - apt-get -y install docker-ce
- apt-get install -y resolvconf linux-headers-$(uname -r) && echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail && systemctl start resolvconf - apt-get install -y resolvconf linux-headers-$(uname -r) && echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail && systemctl start resolvconf
- DEBIAN_FRONTEND=noninteractive apt-get upgrade -y - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
- n=1; while [ $n -le ${k3s_per_node} ]; do docker run -d --restart=unless-stopped -e K3S_URL=https://${k3s_url}:6443 -e K3S_CLUSTER_SECRET="${k3s_cluster_secret}" --privileged --mount type=tmpfs,destination=/var/run --mount type=tmpfs,destination=/run -m 1g --cpus=".7" rancher/k3s:${install_k3s_version}; n=$(( n + 1 )); done - n=1; while [ $n -le ${k3s_per_node} ]; do docker run -d --restart=unless-stopped -e K3S_URL=https://${k3s_url}:6443 -e K3S_CLUSTER_SECRET="${k3s_cluster_secret}" --privileged --mount type=tmpfs,destination=/var/run --mount type=tmpfs,destination=/run -m 1g --cpus=".7" rancher/k3s:${install_k3s_version}; n=$(( n + 1 )); done

2
load_testing/k3s/pool/main.tf
View File

@ -55,7 +55,7 @@ module "k3s-pool-worker-asg" {
asg_name = local.name asg_name = local.name
instance_type = var.worker_instance_type instance_type = var.worker_instance_type
image_id = data.aws_ami.ubuntu.id image_id = data.aws_ami.ubuntu.id
user_data = data.template_file.k3s-pool-worker-user_data.rendered user_data = base64encode(templatefile("${path.module}/files/pool_worker_userdata.tmpl", { k3s_url = data.terraform_remote_state.server.outputs.public_ip[0], k3s_cluster_secret = local.k3s_cluster_secret, install_k3s_version = local.install_k3s_version, k3s_per_node = var.k3s_per_node, extra_ssh_keys = var.extra_ssh_keys }))
ebs_optimized = true ebs_optimized = true
desired_capacity = var.node_count desired_capacity = var.node_count

6
load_testing/k3s/pool/variables.tf
View File

@ -14,3 +14,9 @@ variable "worker_instance_type" {
type = string type = string
default = "c5.4xlarge" default = "c5.4xlarge"
} }
variable "extra_ssh_keys" {
type = list
default = []
description = "Extra ssh keys to inject into Rancher instances"
}

36
load_testing/k3s/server/data.tf
View File

@ -45,39 +45,3 @@ data "template_file" "k3s-prom-yaml" {
graf_host = var.graf_host graf_host = var.graf_host
} }
} }
data "template_file" "k3s-server-user_data" {
template = file("${path.module}/files/server_userdata.tmpl")
vars = {
create_eip = 1
metrics_yaml = base64encode(data.template_file.metrics.rendered)
prom_yaml = base64encode(data.template_file.k3s-prom-yaml.rendered)
eip = join(",", aws_eip.k3s-server.*.public_ip)
k3s_cluster_secret = local.k3s_cluster_secret
install_k3s_version = local.install_k3s_version
k3s_server_args = var.k3s_server_args
}
}
data "template_file" "k3s-prom-worker-user_data" {
template = file("${path.module}/files/worker_userdata.tmpl")
vars = {
k3s_url = aws_eip.k3s-server.0.public_ip
k3s_cluster_secret = local.k3s_cluster_secret
install_k3s_version = local.install_k3s_version
k3s_exec = "--node-label prom=true"
}
}
data "template_file" "k3s-worker-user_data" {
template = file("${path.module}/files/worker_userdata.tmpl")
vars = {
k3s_url = aws_eip.k3s-server.0.public_ip
k3s_cluster_secret = local.k3s_cluster_secret
install_k3s_version = local.install_k3s_version
k3s_exec = ""
}
}

78
load_testing/k3s/server/files/server_userdata.tmpl
View File

@ -1,45 +1,41 @@
#cloud-config #cloud-config
%{ if length(extra_ssh_keys) > 0 }
ssh_authorized_keys: ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC06Qvs+Y9JiyOTeYNGAN/Ukq7SmeCTr7EreD1K8Lwu5VuOmo+SBZh685tNTEGV044HgFvGEOBVreDlO2ArYuwHjUBGnpQGV8/abjoeLrmZBdREAUzBQ1h2GFE/WssKUfum81cnigRK1J3tWP7emq/Y2h/Zw5F09yiCIlXMBX2auKWUCXqwG3xKTi1NVSF9N6BGyFolrAR0LZJ6k7UBXPRc/QDTclI427gSJNbnmn8LVym6YxacV/V9Y7s23iR5zYbhLPe9VJWYNk1brVvfUVb3mILVVYz76KGEq8SHdWlPQPCOp+fSJ+PezDRklnex/MmvhNrBOmMSNcpj7wSLA3hD wmaxwell@wmaxwell-laptop %{ for ssh_key in extra_ssh_keys }
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak - ${ssh_key}
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2TBZGjE+J8ag11dzkFT58J3XPONDrVmalCNrKxsfADfyy0eqdZrG8hAcxAR/5zuj90Gin2uB4RSw6Cn4VHsPZcFpXyQCj1KQDADj+WcuhpXOIOY3AB0LZBly9NI0ll+8lo3QtEaoyRLtrMBhQ6Mooy2M3MTG4JNwU9o3yInuqZWf9PvtW6KxMl+ygg1xZkljhemGZ9k0wSrjqif+8usNbzVlCOVQmZwZA+BZxbdcLNwkg7zWJSXzDIXyqM6iWPGXQDEbWLq3+HR1qKucTCSxjbqoe0FD5xcW7NHIME5XKX84yH92n6yn+rxSsyUfhJWYqJd+i0fKf5UbN6qLrtd/D darren@darrens %{ endfor }
%{ endif }
write_files: write_files:
- path: /var/lib/rancher/k3s/server/manifests/metrics.yaml - path: /var/lib/rancher/k3s/server/manifests/metrics.yaml
permissions: "0755" permissions: "0755"
owner: root:root owner: root:root
encoding: b64 encoding: b64
content: ${metrics_yaml} content: ${metrics_yaml}
- path: /var/lib/rancher/k3s/server/manifests/prom.yaml - path: /var/lib/rancher/k3s/server/manifests/prom.yaml
permissions: "0755" permissions: "0755"
owner: root:root owner: root:root
encoding: b64 encoding: b64
content: ${prom_yaml} content: ${prom_yaml}
runcmd: runcmd:
- echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf
- echo "fs.file-max = 12000500" >> /etc/sysctl.conf - echo "fs.file-max = 12000500" >> /etc/sysctl.conf
- echo "fs.nr_open = 20000500" >> /etc/sysctl.conf - echo "fs.nr_open = 20000500" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf
- echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf - echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf
- echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf - echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf
- ulimit -n 20000000 - ulimit -n 20000000
- echo "# <domain> <type> <item> <value>" >> /etc/security/limits.d/limits.conf - echo "# <domain> <type> <item> <value>" >> /etc/security/limits.d/limits.conf
- echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf - echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf
- echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf - echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf
- sysctl -p - sysctl -p
- apt-get update - apt-get update
- apt-get install -y software-properties-common resolvconf linux-headers-$(uname -r) - apt-get install -y software-properties-common resolvconf linux-headers-$(uname -r)
- echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail - echo "nameserver 1.1.1.1" > /etc/resolvconf/resolv.conf.d/tail
- systemctl start resolvconf - systemctl start resolvconf
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - until (curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="${k3s_server_args} --disable-agent --no-deploy traefik --no-deploy servicelb --cluster-cidr=10.0.0.0/8 --service-cidr=192.168.0.0/16 --cluster-dns=192.168.0.10 --tls-san ${public_ip}" K3S_CLUSTER_SECRET="${k3s_cluster_secret}" INSTALL_K3S_VERSION=${install_k3s_version} sh -); do echo 'Error installing k3s'; sleep 1; done
- add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- apt-get update
- apt-get -y install docker-ce
- DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
- if [ "${create_eip}" = "1" ]; then docker run -e "EIP=${eip}" cloudnautique/eip-autoassign:latest; fi
- until (curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="${k3s_server_args} --disable-agent --no-deploy traefik --no-deploy servicelb --cluster-cidr=10.0.0.0/8 --service-cidr=192.168.0.0/16 --cluster-dns=192.168.0.10 --tls-san ${eip}" K3S_CLUSTER_SECRET="${k3s_cluster_secret}" INSTALL_K3S_VERSION=${install_k3s_version} sh -); do echo 'Error installing k3s'; sleep 1; done

46
load_testing/k3s/server/files/worker_userdata.tmpl
View File

@ -1,26 +1,26 @@
#cloud-config #cloud-config
%{ if length(extra_ssh_keys) > 0 }
ssh_authorized_keys: ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC06Qvs+Y9JiyOTeYNGAN/Ukq7SmeCTr7EreD1K8Lwu5VuOmo+SBZh685tNTEGV044HgFvGEOBVreDlO2ArYuwHjUBGnpQGV8/abjoeLrmZBdREAUzBQ1h2GFE/WssKUfum81cnigRK1J3tWP7emq/Y2h/Zw5F09yiCIlXMBX2auKWUCXqwG3xKTi1NVSF9N6BGyFolrAR0LZJ6k7UBXPRc/QDTclI427gSJNbnmn8LVym6YxacV/V9Y7s23iR5zYbhLPe9VJWYNk1brVvfUVb3mILVVYz76KGEq8SHdWlPQPCOp+fSJ+PezDRklnex/MmvhNrBOmMSNcpj7wSLA3hD wmaxwell@wmaxwell-laptop %{ for ssh_key in extra_ssh_keys }
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5O7k6gRYCU7YPkCH6dyXVW10izMAkDAQtQxNxdRE22 drpebcak - ${ssh_key}
%{ endfor }
%{ endif }
runcmd: runcmd:
- echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_interval = 3600" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_stale_time = 3600" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf
- echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf - echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf
- echo "fs.file-max = 12000500" >> /etc/sysctl.conf - echo "fs.file-max = 12000500" >> /etc/sysctl.conf
- echo "fs.nr_open = 20000500" >> /etc/sysctl.conf - echo "fs.nr_open = 20000500" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_mem = '10000000 10000000 10000000'" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_rmem = '1024 4096 16384'" >> /etc/sysctl.conf
- echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf - echo "net.ipv4.tcp_wmem = '1024 4096 16384'" >> /etc/sysctl.conf
- echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf - echo "net.core.rmem_max = 16384" >> /etc/sysctl.conf
- echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf - echo "net.core.wmem_max = 16384" >> /etc/sysctl.conf
- ulimit -n 20000000 - ulimit -n 20000
- echo "# <domain> <type> <item> <value>" >> /etc/security/limits.d/limits.conf - echo "# <domain> <type> <item> <value>" >> /etc/security/limits.d/limits.conf
- echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf - echo " * soft nofile 20000" >> /etc/security/limits.d/limits.conf
- echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf - echo " * hard nofile 20000" >> /etc/security/limits.d/limits.conf
- sysctl -p - sysctl -p
- apt-get update - until (curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${install_k3s_version} INSTALL_K3S_EXEC="${k3s_exec}" K3S_URL=https://${k3s_url}:6443 K3S_CLUSTER_SECRET="${k3s_cluster_secret}" sh -); do echo 'k3s did not install correctly'; sleep 1; done
- apt-get install -y software-properties-common
- DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
- until (curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${install_k3s_version} INSTALL_K3S_EXEC="${k3s_exec}" K3S_URL=https://${k3s_url}:6443 K3S_CLUSTER_SECRET="${k3s_cluster_secret}" sh -); do echo 'k3s did not install correctly'; sleep 1; done

58
load_testing/k3s/server/iam.tf
View File

@ -1,58 +0,0 @@
resource "aws_iam_instance_profile" "k3s-server" {
name_prefix = "load-testing-k3s-server"
role = aws_iam_role.k3s-server.name
lifecycle {
create_before_destroy = true
}
}
resource "aws_iam_role" "k3s-server" {
name_prefix = "load-testing-k3s-server"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
lifecycle {
create_before_destroy = true
}
}
resource "aws_iam_role_policy" "k3s-server" {
name_prefix = "load-testing-k3s-server"
role = aws_iam_role.k3s-server.id
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"ec2:AssociateAddress",
"ec2:DescribeAddresses"
],
"Sid": "",
"Resource": [
"*"
],
"Effect": "Allow"
}
]
}
EOF
}

75
load_testing/k3s/server/main.tf
View File

@ -5,12 +5,10 @@ terraform {
} }
locals { locals {
name = "k3s-load-server" name = var.name
node_count = 1 k3s_cluster_secret = var.k3s_cluster_secret
k3s_cluster_secret = "pvc-6476dcaf-73a0-11e9-b8e5-06943b744282" install_k3s_version = var.k3s_version
install_k3s_version = "v0.9.0-rc2" prom_worker_node_count = var.prom_worker_node_count
prom_worker_node_count = 0
worker_node_count = 0
} }
provider "aws" { provider "aws" {
@ -18,13 +16,8 @@ provider "aws" {
profile = "rancher-eng" profile = "rancher-eng"
} }
resource "aws_eip" "k3s-server" {
count = local.node_count
vpc = true
}
resource "aws_security_group" "k3s" { resource "aws_security_group" "k3s" {
name = "${local.name}-rancher-server" name = "${local.name}-sg"
vpc_id = data.aws_vpc.default.id vpc_id = data.aws_vpc.default.id
ingress { ingress {
@ -63,48 +56,36 @@ resource "aws_security_group" "k3s" {
} }
} }
module "k3s-server-asg" { resource "aws_spot_instance_request" "k3s-server" {
source = "terraform-aws-modules/autoscaling/aws" instance_type = var.server_instance_type
version = "3.0.0" ami = data.aws_ami.ubuntu.id
name = "load-testing-k3s-server" user_data = base64encode(templatefile("${path.module}/files/server_userdata.tmpl", { extra_ssh_keys = var.extra_ssh_keys, public_ip = aws_spot_instance_request.k3s-server.public_ip, metrics_yaml = base64encode(data.template_file.metrics.rendered), prom_yaml = base64encode(data.template_file.k3s-prom-yaml.rendered), k3s_cluster_secret = local.k3s_cluster_secret, install_k3s_version = local.install_k3s_version, k3s_server_args = var.k3s_server_args }))
asg_name = "load-testing-k3s-server"
instance_type = var.server_instance_type
image_id = data.aws_ami.ubuntu.id
user_data = data.template_file.k3s-server-user_data.rendered
ebs_optimized = true ebs_optimized = true
iam_instance_profile = aws_iam_instance_profile.k3s-server.name wait_for_fulfillment = true
desired_capacity = local.node_count
health_check_type = "EC2"
max_size = local.node_count
min_size = local.node_count
vpc_zone_identifier = [data.aws_subnet.selected.id]
spot_price = "1.591"
security_groups = [ security_groups = [
aws_security_group.k3s.id, aws_security_group.k3s.id,
] ]
lc_name = "load-testing-k3s-server" root_block_device {
volume_size = "1000"
volume_type = "gp2"
}
root_block_device = [ tags = {
{ Name = "${local.name}-server"
volume_size = "1000" }
volume_type = "gp2"
},
]
} }
module "k3s-prom-worker-asg" { module "k3s-prom-worker-asg" {
source = "terraform-aws-modules/autoscaling/aws" source = "terraform-aws-modules/autoscaling/aws"
version = "3.0.0" version = "3.0.0"
name = "load-testing-k3s-prom-worker" name = "${local.name}-prom-worker"
asg_name = "load-testing-k3s-prom-worker" asg_name = "${local.name}-prom-worker"
instance_type = "m5.large" instance_type = "m5.large"
image_id = data.aws_ami.ubuntu.id image_id = data.aws_ami.ubuntu.id
user_data = data.template_file.k3s-prom-worker-user_data.rendered user_data = base64encode(templatefile("${path.module}/files/worker_userdata.tmpl", { extra_ssh_keys = var.extra_ssh_keys, k3s_url = aws_spot_instance_request.k3s-server.public_ip, k3s_cluster_secret = local.k3s_cluster_secret, install_k3s_version = local.install_k3s_version, k3s_exec = "--node-label prom=true" }))
ebs_optimized = true ebs_optimized = true
iam_instance_profile = aws_iam_instance_profile.k3s-server.name
desired_capacity = local.prom_worker_node_count desired_capacity = local.prom_worker_node_count
health_check_type = "EC2" health_check_type = "EC2"
@ -117,7 +98,7 @@ module "k3s-prom-worker-asg" {
aws_security_group.k3s.id, aws_security_group.k3s.id,
] ]
lc_name = "load-testing-k3s-prom-worker" lc_name = "${local.name}-prom-worker"
root_block_device = [ root_block_device = [
{ {
@ -130,6 +111,6 @@ module "k3s-prom-worker-asg" {
resource "null_resource" "get-kubeconfig" { resource "null_resource" "get-kubeconfig" {
provisioner "local-exec" { provisioner "local-exec" {
interpreter = ["bash", "-c"] interpreter = ["bash", "-c"]
command = "until ssh ubuntu@${aws_eip.k3s-server.0.public_ip} 'sudo sed \"s/localhost/${aws_eip.k3s-server.0.public_ip}/g;s/127.0.0.1/${aws_eip.k3s-server.0.public_ip}/g\" /etc/rancher/k3s/k3s.yaml' >| ../cluster-loader/kubeConfig.yaml; do sleep 5; done" command = "until ssh ubuntu@${aws_spot_instance_request.k3s-server.public_ip} 'sudo sed \"s/localhost/$aws_spot_instance_request.k3s-server.public_ip}/g;s/127.0.0.1/${aws_spot_instance_request.k3s-server.public_ip}/g\" /etc/rancher/k3s/k3s.yaml' >| ../cluster-loader/kubeConfig.yaml; do sleep 5; done"
} }
} }

2
load_testing/k3s/server/outputs.tf
View File

@ -1,5 +1,5 @@
output "public_ip" { output "public_ip" {
value = aws_eip.k3s-server.*.public_ip value = aws_spot_instance_request.k3s-server.public_ip
} }
output "install_k3s_version" { output "install_k3s_version" {

26
load_testing/k3s/server/variables.tf
View File

@ -1,12 +1,38 @@
variable "server_instance_type" { variable "server_instance_type" {
# default = "c4.8xlarge" # default = "c4.8xlarge"
} }
variable "k3s_version" {
default = "v0.9.1"
type = string
description = "Version of K3S to install"
}
variable "k3s_server_args" { variable "k3s_server_args" {
default = "" default = ""
} }
variable "prom_worker_node_count" {
default = 0
type = number
description = "The number of workers to create labeled for prometheus"
}
variable "k3s_cluster_secret" {
default = "pvc-6476dcaf-73a0-11e9-b8e5-06943b744282"
type = string
description = "Cluster secret for k3s cluster registration"
}
variable "prom_host" { variable "prom_host" {
default = "" default = ""
} }
variable "graf_host" { variable "graf_host" {
default = "" default = ""
} }
variable "name" {
default = "k3s-loadtest"
type = string
description = "Name to identify this cluster"
}
variable "extra_ssh_keys" {
type = list
default = []
description = "Extra ssh keys to inject into Rancher instances"
}