[K3s][Windows Port] Build script, multi-call binary, and Flannel (#7259)

* initial windows port.

Signed-off-by: Sean Yen <seanyen@microsoft.com>
Signed-off-by: Derek Nola <derek.nola@suse.com>
Co-authored-by: Derek Nola <derek.nola@suse.com>
Co-authored-by: Wei Ran <weiran@microsoft.com>

cmd/k3s/main.go

@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/configfilearg"
@@ -46,8 +45,8 @@ func main() {
 	app := cmds.NewApp()
 	app.EnableBashCompletion = true
 	app.Commands = []cli.Command{
-		cmds.NewServerCommand(internalCLIAction(version.Program+"-server", dataDir, os.Args)),
+		cmds.NewServerCommand(internalCLIAction(version.Program+"-server"+programPostfix, dataDir, os.Args)),
-		cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent", dataDir, os.Args)),
+		cmds.NewAgentCommand(internalCLIAction(version.Program+"-agent"+programPostfix, dataDir, os.Args)),
 		cmds.NewKubectlCommand(externalCLIAction("kubectl", dataDir)),
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
@@ -157,7 +156,7 @@ func externalCLI(cli, dataDir string, args []string) error {
 			os.Setenv("CRI_CONFIG_FILE", findCriConfig(dataDir))
 		}
 	}
-	return stageAndRun(dataDir, cli, append([]string{cli}, args...))
+	return stageAndRun(dataDir, cli, append([]string{cli}, args...), false)
 }
 
 // internalCLIAction returns a function that will call a K3s internal command, be used as the Action of a cli.Command.
@@ -173,11 +172,11 @@ func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context
 
 // stageAndRunCLI calls an external binary.
 func stageAndRunCLI(cli *cli.Context, cmd string, dataDir string, args []string) error {
-	return stageAndRun(dataDir, cmd, args)
+	return stageAndRun(dataDir, cmd, args, true)
 }
 
 // stageAndRun does the actual work of setting up and calling an external binary.
-func stageAndRun(dataDir, cmd string, args []string) error {
+func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) error {
 	dir, err := extract(dataDir)
 	if err != nil {
 		return errors.Wrap(err, "extracting data")
@@ -186,9 +185,9 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 
 	var pathEnv string
 	if findPreferBundledBin(args) {
-		pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH")
+		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH")
 	} else {
-		pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux")
+		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux")
 	}
 	if err := os.Setenv("PATH", pathEnv); err != nil {
 		return err
@@ -204,10 +203,7 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 
 	logrus.Debugf("Running %s %v", cmd, args)
 
-	if err := syscall.Exec(cmd, args, os.Environ()); err != nil {
+	return runExec(cmd, args, calledAsInternal)
-		return errors.Wrapf(err, "exec %s failed", cmd)
-	}
-	return nil
 }
 
 // getAssetAndDir returns the name of the bindata asset, along with a directory path
@@ -223,7 +219,7 @@ func getAssetAndDir(dataDir string) (string, string) {
 func extract(dataDir string) (string, error) {
 	// check if content already exists in requested data-dir
 	asset, dir := getAssetAndDir(dataDir)
-	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s")); err == nil {
+	if _, err := os.Stat(filepath.Join(dir, "bin", "k3s"+programPostfix)); err == nil {
 		return dir, nil
 	}
 
@@ -232,7 +228,7 @@ func extract(dataDir string) (string, error) {
 	// dir if the assets already exist in the default path.
 	if dataDir != datadir.DefaultDataDir {
 		_, defaultDir := getAssetAndDir(datadir.DefaultDataDir)
-		if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s")); err == nil {
+		if _, err := os.Stat(filepath.Join(defaultDir, "bin", "k3s"+programPostfix)); err == nil {
 			return defaultDir, nil
 		}
 	}

cmd/k3s/main_linux.go

@@ -0,0 +1,20 @@
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"os"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+const programPostfix = ""
+
+func runExec(cmd string, args []string, calledAsInternal bool) (err error) {
+	if err := syscall.Exec(cmd, args, os.Environ()); err != nil {
+		return errors.Wrapf(err, "exec %s failed", cmd)
+	}
+	return nil
+}

cmd/k3s/main_windows.go

@@ -0,0 +1,24 @@
+//go:build windows
+// +build windows
+
+package main
+
+import (
+	"os"
+	"os/exec"
+)
+
+const programPostfix = ".exe"
+
+func runExec(cmd string, args []string, calledAsInternal bool) (err error) {
+	// syscall.Exec: not supported by windows
+	if calledAsInternal {
+		args = args[1:]
+	}
+	cmdObj := exec.Command(cmd, args...)
+	cmdObj.Stdout = os.Stdout
+	cmdObj.Stderr = os.Stderr
+	cmdObj.Stdin = os.Stdin
+	cmdObj.Env = os.Environ()
+	return cmdObj.Run()
+}

go.mod

@@ -86,6 +86,7 @@ replace (
 )
 
 require (
+	github.com/Microsoft/hcsshim v0.11.1
 	github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000
@@ -184,7 +185,6 @@ require (
 	github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/Microsoft/hcsshim v0.11.1 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/Rican7/retry v0.1.0 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect

pkg/agent/flannel/setup.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"path/filepath"
+	goruntime "runtime"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/agent/util"
@@ -23,34 +24,6 @@ import (
 )
 
 const (
-	cniConf = `{
-  "name":"cbr0",
-  "cniVersion":"1.0.0",
-  "plugins":[
-    {
-      "type":"flannel",
-      "delegate":{
-        "hairpinMode":true,
-        "forceAddress":true,
-        "isDefaultGateway":true
-      }
-    },
-    {
-      "type":"portmap",
-      "capabilities":{
-        "portMappings":true
-      }
-    },
-    {
-      "type":"bandwidth",
-      "capabilities":{
-        "bandwidth":true
-      }
-    }
-  ]
-}
-`
-
 	flannelConf = `{
 	"Network": "%CIDR%",
 	"EnableIPv6": %IPV6_ENABLED%,
@@ -60,10 +33,6 @@ const (
 }
 `
 
-	vxlanBackend = `{
-	"Type": "vxlan"
-}`
-
 	hostGWBackend = `{
 	"Type": "host-gw"
 }`
@@ -153,7 +122,20 @@ func createCNIConf(dir string, nodeConfig *config.Node) error {
 		logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile)
 		return util.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p, false)
 	}
-	return util.WriteFile(p, cniConf)
+
+	cniConfJSON := cniConf
+	if goruntime.GOOS == "windows" {
+		extIface, err := LookupExtInterface(nodeConfig.FlannelIface, ipv4)
+		if err != nil {
+			return err
+		}
+
+		cniConfJSON = strings.ReplaceAll(cniConfJSON, "%IPV4_ADDRESS%", extIface.IfaceAddr.String())
+		cniConfJSON = strings.ReplaceAll(cniConfJSON, "%CLUSTER_CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
+		cniConfJSON = strings.ReplaceAll(cniConfJSON, "%SERVICE_CIDR%", nodeConfig.AgentConfig.ServiceCIDR.String())
+	}
+
+	return util.WriteFile(p, cniConfJSON)
 }
 
 func createFlannelConf(nodeConfig *config.Node) error {
@@ -205,6 +187,16 @@ func createFlannelConf(nodeConfig *config.Node) error {
 	var backendConf string
 	backendOptions := make(map[string]string)
 
+	// precheck and error out unsupported flannel backends.
+	switch nodeConfig.FlannelBackend {
+	case config.FlannelBackendHostGW:
+	case config.FlannelBackendTailscale:
+	case config.FlannelBackendWireguardNative:
+		if goruntime.GOOS == "windows" {
+			return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.FlannelBackend)
+		}
+	}
+
 	switch nodeConfig.FlannelBackend {
 	case config.FlannelBackendVXLAN:
 		backendConf = vxlanBackend

pkg/agent/flannel/setup_linux.go

@@ -0,0 +1,38 @@
+//go:build linux
+// +build linux
+
+package flannel
+
+const (
+	cniConf = `{
+  "name":"cbr0",
+  "cniVersion":"1.0.0",
+  "plugins":[
+    {
+      "type":"flannel",
+      "delegate":{
+        "hairpinMode":true,
+        "forceAddress":true,
+        "isDefaultGateway":true
+      }
+    },
+    {
+      "type":"portmap",
+      "capabilities":{
+        "portMappings":true
+      }
+    },
+    {
+      "type":"bandwidth",
+      "capabilities":{
+        "bandwidth":true
+      }
+    }
+  ]
+}
+`
+
+	vxlanBackend = `{
+	"Type": "vxlan"
+}`
+)

pkg/agent/flannel/setup_windows.go

@@ -0,0 +1,59 @@
+//go:build windows
+// +build windows
+
+package flannel
+
+const (
+	cniConf = `{
+  "name":"flannel.4096",
+  "cniVersion":"1.0.0",
+  "plugins":[
+    {
+      "type":"flannel",
+      "capabilities": {
+        "portMappings": true,
+        "dns": true
+      },
+      "delegate": {
+        "type": "win-overlay",
+        "apiVersion": 2,
+        "Policies": [{
+            "Name": "EndpointPolicy",
+            "Value": {
+                "Type": "OutBoundNAT",
+                "Settings": {
+                  "Exceptions": [
+                    "%CLUSTER_CIDR%", "%SERVICE_CIDR%"
+                  ]
+                }
+            }
+        }, {
+            "Name": "EndpointPolicy",
+            "Value": {
+                "Type": "SDNRoute",
+                "Settings": {
+                  "DestinationPrefix": "%SERVICE_CIDR%",
+                  "NeedEncap": true
+                }
+            }
+        }, {
+            "name": "EndpointPolicy",
+            "value": {
+                "Type": "ProviderAddress",
+                "Settings": {
+                    "ProviderAddress": "%IPV4_ADDRESS%"
+                }
+            }
+        }]
+      }
+    }
+  ]
+}
+`
+
+	vxlanBackend = `{
+	"Type": "vxlan",
+	"VNI": 4096,
+	"Port": 4789
+}`
+)

pkg/agent/run.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	goruntime "runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -83,6 +84,11 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	enableIPv6 := dualCluster || clusterIPv6
 	enableIPv4 := dualCluster || clusterIPv4
 
+	// dualStack or IPv6 are not supported on Windows node
+	if (goruntime.GOOS == "windows") && enableIPv6 {
+		return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node")
+	}
+
 	conntrackConfig, err := getConntrackConfig(nodeConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")

pkg/cli/crictl/crictl.go

@@ -1,11 +1,17 @@
 package crictl
 
 import (
+	"os"
+	"runtime"
+
 	"github.com/kubernetes-sigs/cri-tools/cmd/crictl"
 	"github.com/urfave/cli"
 )
 
 func Run(ctx *cli.Context) error {
+	if runtime.GOOS == "windows" {
+		os.Args = os.Args[1:]
+	}
 	crictl.Main()
 	return nil
 }

pkg/containerd/builtins_windows.go

@@ -0,0 +1,27 @@
+//go:build ctrd
+// +build ctrd
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package containerd
+
+import (
+	_ "github.com/containerd/containerd/diff/lcow"
+	_ "github.com/containerd/containerd/diff/windows"
+	_ "github.com/containerd/containerd/snapshots/lcow"
+	_ "github.com/containerd/containerd/snapshots/windows"
+)

pkg/daemons/agent/agent_windows.go

@@ -122,5 +122,6 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
 	if cfg.ProtectKernelDefaults {
 		argsMap["protect-kernel-defaults"] = "true"
 	}
+
 	return argsMap
 }

pkg/daemons/executor/embed.go

@@ -96,9 +96,9 @@ func (e *Embedded) Kubelet(ctx context.Context, args []string) error {
 	return nil
 }
 
-func (*Embedded) KubeProxy(ctx context.Context, args []string) error {
+func (e *Embedded) KubeProxy(ctx context.Context, args []string) error {
 	command := proxy.NewProxyCommand()
-	command.SetArgs(args)
+	command.SetArgs(daemonconfig.GetArgs(platformKubeProxyArgs(e.nodeConfig), args))
 
 	go func() {
 		defer func() {

pkg/daemons/executor/embed_linux.go

@@ -0,0 +1,16 @@
+//go:build linux && !no_embedded_executor
+// +build linux,!no_embedded_executor
+
+package executor
+
+import (
+	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
+
+	// registering k3s cloud provider
+	_ "github.com/k3s-io/k3s/pkg/cloudprovider"
+)
+
+func platformKubeProxyArgs(nodeConfig *daemonconfig.Node) map[string]string {
+	argsMap := map[string]string{}
+	return argsMap
+}

pkg/daemons/executor/embed_windows.go

@@ -0,0 +1,90 @@
+//go:build windows && !no_embedded_executor
+// +build windows,!no_embedded_executor
+
+package executor
+
+import (
+	"encoding/json"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/Microsoft/hcsshim"
+	"github.com/sirupsen/logrus"
+
+	// registering k3s cloud provider
+	_ "github.com/k3s-io/k3s/pkg/cloudprovider"
+	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
+)
+
+const (
+	networkName = "flannel.4096"
+)
+
+type SourceVipResponse struct {
+	IP4 struct {
+		IP string `json:"ip"`
+	} `json:"ip4"`
+}
+
+func platformKubeProxyArgs(nodeConfig *daemonconfig.Node) map[string]string {
+	argsMap := map[string]string{}
+	argsMap["network-name"] = networkName
+	if sourceVip := waitForSourceVip(networkName, nodeConfig); sourceVip != "" {
+		argsMap["source-vip"] = sourceVip
+	}
+	return argsMap
+}
+
+func waitForSourceVip(networkName string, nodeConfig *daemonconfig.Node) string {
+	for range time.Tick(time.Second * 5) {
+		network, err := hcsshim.GetHNSNetworkByName(networkName)
+		if err != nil {
+			logrus.WithError(err).Warningf("can't find HNS network, retrying %s", networkName)
+			continue
+		}
+		if network.ManagementIP == "" {
+			logrus.WithError(err).Warningf("wait for management IP, retrying %s", networkName)
+			continue
+		}
+
+		subnet := network.Subnets[0].AddressPrefix
+
+		configData := `{
+			"cniVersion": "0.2.0",
+			"name": "vxlan0",
+			"ipam": {
+				"type": "host-local",
+				"ranges": [[{"subnet":"` + subnet + `"}]],
+				"dataDir": "/var/lib/cni/networks"
+			}
+		}`
+
+		cmd := exec.Command("host-local.exe")
+		cmd.Env = append(os.Environ(),
+			"CNI_COMMAND=ADD",
+			"CNI_CONTAINERID=dummy",
+			"CNI_NETNS=dummy",
+			"CNI_IFNAME=dummy",
+			"CNI_PATH="+nodeConfig.AgentConfig.CNIBinDir,
+		)
+
+		cmd.Stdin = strings.NewReader(configData)
+		out, err := cmd.Output()
+		if err != nil {
+			logrus.WithError(err).Warning("Failed to execute host-local.exe")
+			continue
+		}
+
+		var sourceVipResp SourceVipResponse
+		err = json.Unmarshal(out, &sourceVipResp)
+		if err != nil {
+			logrus.WithError(err).Warning("Failed to unmarshal sourceVip response")
+			continue
+		}
+
+		return strings.TrimSpace(strings.Split(sourceVipResp.IP4.IP, "/")[0])
+	}
+	return ""
+}

pkg/kubectl/main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"runtime"
 	"strings"
 	"time"
 
@@ -15,6 +16,9 @@ import (
 )
 
 func Main() {
+	if runtime.GOOS == "windows" {
+		os.Args = os.Args[1:]
+	}
 	kubenv := os.Getenv("KUBECONFIG")
 	for i, arg := range os.Args {
 		if strings.HasPrefix(arg, "--kubeconfig=") {

scripts/build

@@ -66,6 +66,10 @@ TAGS="ctrd apparmor seccomp netcgo osusergo providerless urfave_cli_no_docs"
 RUNC_TAGS="apparmor seccomp"
 RUNC_STATIC="static"
 
+if [ ${OS} = windows ]; then
+    TAGS="ctrd netcgo osusergo providerless"
+fi
+
 if [ "$SELINUX" = "true" ]; then
     TAGS="$TAGS selinux"
     RUNC_TAGS="$RUNC_TAGS selinux"
@@ -97,21 +101,41 @@ if [ ${ARCH} = s390x ]; then
     export GOARCH="s390x"
 fi
 
-rm -f \
+k3s_binaries=(
-    bin/k3s-agent \
+    "bin/k3s-agent"
-    bin/k3s-server \
+    "bin/k3s-server"
-    bin/k3s-token \
+    "bin/k3s-token"
-    bin/k3s-etcd-snapshot \
+    "bin/k3s-etcd-snapshot"
-    bin/k3s-secrets-encrypt \
+    "bin/k3s-secrets-encrypt"
-    bin/k3s-certificate \
+    "bin/k3s-certificate"
-    bin/k3s-completion \
+    "bin/k3s-completion"
-    bin/kubectl \
+    "bin/kubectl"
-    bin/crictl \
+    "bin/containerd"
-    bin/ctr \
+    "bin/crictl"
-    bin/containerd \
+    "bin/ctr"
-    bin/containerd-shim \
+)
-    bin/containerd-shim-runc-v2 \
+
-    bin/runc
+containerd_binaries=(
+    "bin/containerd-shim"
+    "bin/containerd-shim-runc-v2"
+    "bin/runc"
+    "bin/containerd-shim-runhcs-v1"
+    "bin/runhcs"
+)
+
+for i in "${k3s_binaries[@]}"; do
+    if [ -f "$i${BINARY_POSTFIX}" ]; then
+        echo "Removing $i${BINARY_POSTFIX}"
+        rm -f "$i${BINARY_POSTFIX}"
+    fi
+done
+
+for i in "${containerd_binaries[@]}"; do
+    if [ -f "$i${BINARY_POSTFIX}" ]; then
+        echo "Removing $i${BINARY_POSTFIX}"
+        rm -f "$i${BINARY_POSTFIX}"
+    fi
+done
 
 cleanup() {
     exit_status=$?
@@ -120,7 +144,7 @@ cleanup() {
 }
 
 INSTALLBIN=$(pwd)/bin
-if [ ! -x ${INSTALLBIN}/cni ]; then
+if [ ! -x ${INSTALLBIN}/cni${BINARY_POSTFIX} ]; then
 (
     echo Building cni
     TMPDIR=$(mktemp -d)
@@ -128,36 +152,46 @@ if [ ! -x ${INSTALLBIN}/cni ]; then
     WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
     git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
     cd $WORKDIR
-    GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni
+    GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni${BINARY_POSTFIX}
 )
 fi
 
 echo Building k3s
-CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server
+CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s${BINARY_POSTFIX} ./cmd/server
-ln -s k3s ./bin/containerd
+
-ln -s k3s ./bin/crictl
+for i in "${k3s_binaries[@]}"; do
-ln -s k3s ./bin/ctr
+    ln -s "k3s${BINARY_POSTFIX}" "$i${BINARY_POSTFIX}"
-ln -s k3s ./bin/k3s-agent
+done
-ln -s k3s ./bin/k3s-certificate
-ln -s k3s ./bin/k3s-completion
-ln -s k3s ./bin/k3s-etcd-snapshot
-ln -s k3s ./bin/k3s-secrets-encrypt
-ln -s k3s ./bin/k3s-server
-ln -s k3s ./bin/k3s-token
-ln -s k3s ./bin/kubectl
 
 export GOPATH=$(pwd)/build
 
-echo Building containerd
+case ${OS} in
-pushd ./build/src/github.com/containerd/containerd
+  linux)
-TAGS="${TAGS/netcgo/netgo}"
+    echo Building containerd-shim
-CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2
+    pushd ./build/src/github.com/containerd/containerd
-popd
+    TAGS="${TAGS/netcgo/netgo}"
-cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/
+    CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2
+    popd
+    cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/
 
-echo Building runc
+    echo Building runc
-pushd ./build/src/github.com/opencontainers/runc
+    pushd ./build/src/github.com/opencontainers/runc
-rm -f runc
+    rm -f runc
-make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC
+    make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC
-popd
+    popd
-cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/
+    cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/
+    ;;
+  windows)
+    echo Building containerd-shim-runhcs-v1
+    pushd ./build/src/github.com/microsoft/hcsshim
+    TAGS="${TAGS/netcgo/netgo}"
+    CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runhcs-v1${BINARY_POSTFIX} ./cmd/containerd-shim-runhcs-v1
+    CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/runhcs${BINARY_POSTFIX} ./cmd/runhcs
+    popd
+    cp -vf ./build/src/github.com/microsoft/hcsshim/bin/*${BINARY_POSTFIX} ./bin/
+    ;;
+  *)
+    echo "[ERROR] unrecognized opertaing system: ${OS}"
+    exit 1
+    ;;
+esac

scripts/download

@@ -10,6 +10,7 @@ CHARTS_URL=https://k3s.io/k3s-charts/assets
 CHARTS_DIR=build/static/charts
 RUNC_DIR=build/src/github.com/opencontainers/runc
 CONTAINERD_DIR=build/src/github.com/containerd/containerd
+HCSSHIM_DIR=build/src/github.com/microsoft/hcsshim
 DATA_DIR=build/data
 export TZ=UTC
 
@@ -17,12 +18,24 @@ umask 022
 rm -rf ${CHARTS_DIR}
 rm -rf ${RUNC_DIR}
 rm -rf ${CONTAINERD_DIR}
+rm -rf ${HCSSHIM_DIR}
 mkdir -p ${CHARTS_DIR}
 mkdir -p ${DATA_DIR}
 
-curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf -
+case ${OS} in
-
+  linux)
-git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
+    git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
+    curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf -
+    cp scripts/wg-add.sh bin/aux
+    ;;
+  windows)
+    git clone --single-branch --branch=${VERSION_HCSSHIM} --depth=1 https://github.com/microsoft/hcsshim ${HCSSHIM_DIR}
+    ;;
+  *)
+    echo "[ERROR] unrecognized opertaing system: ${OS}"
+    exit 1
+    ;;
+esac
 
 git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://${PKG_CONTAINERD_K3S} ${CONTAINERD_DIR}
 
@@ -30,5 +43,3 @@ for CHART_FILE in $(grep -rlF HelmChart manifests/ | xargs yq eval --no-doc .spe
   CHART_NAME=$(echo $CHART_FILE | grep -oE '^(-*[a-z])+')
   curl -sfL ${CHARTS_URL}/${CHART_NAME}/${CHART_FILE} -o ${CHARTS_DIR}/${CHART_FILE}
 done
-
-cp scripts/wg-add.sh bin/aux

scripts/package-cli

@@ -8,13 +8,31 @@ cd $(dirname $0)/..
 GO=${GO-go}
 
 for i in containerd crictl kubectl k3s-agent k3s-server k3s-token k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s-completion; do
-    rm -f bin/$i
+    rm -f bin/$i${BINARY_POSTFIX}
-    ln -s k3s bin/$i
+    ln -s k3s${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX}
 done
 
-for i in bandwidth bridge firewall flannel host-local loopback portmap; do
+cni_binaries=(
-    rm -f bin/$i
+    "bandwidth"
-    ln -s cni bin/$i
+    "bridge"
+    "firewall"
+    "flannel"
+    "host-local"
+    "loopback"
+    "portmap"
+)
+
+if [ ${OS} = windows ]; then
+    cni_binaries=(
+        "win-overlay"
+        "flannel"
+        "host-local"
+    )
+fi
+
+for i in "${cni_binaries[@]}"; do
+    rm -f bin/$i${BINARY_POSTFIX}
+    ln -s cni${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX}
 done
 
 cp contrib/util/check-config.sh bin/check-config
@@ -22,6 +40,7 @@ cp contrib/util/check-config.sh bin/check-config
 rm -rf build/data
 mkdir -p build/data build/out
 mkdir -p dist/artifacts
+mkdir -p ./etc
 
 (
     set +x
@@ -50,9 +69,9 @@ elif [ ${ARCH} = s390x ]; then
     BIN_SUFFIX="-s390x"
 fi
 
-CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}
+CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}${BINARY_POSTFIX}
 
-"${GO}" generate
+GOOS=linux CC=gcc CXX=g++ "${GO}" generate
 LDFLAGS="
     -X github.com/k3s-io/k3s/pkg/version.Version=$VERSION
     -X github.com/k3s-io/k3s/pkg/version.GitCommit=${COMMIT:0:8}
@@ -60,7 +79,7 @@ LDFLAGS="
 "
 TAGS="urfave_cli_no_docs"
 STATIC="-extldflags '-static'"
-CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
+CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -buildvcs=false -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s
 
 stat ${CMD_NAME}
 

scripts/validate

@@ -15,7 +15,7 @@ echo Running: go mod tidy
 go mod tidy
 
 echo Running: go generate
-go generate
+GOOS=linux CC=gcc CXX=g++ go generate
 
 echo Running validation
 

scripts/version.sh

@@ -2,6 +2,7 @@
 
 GO=${GO-go}
 ARCH=${ARCH:-$("${GO}" env GOARCH)}
+OS=${OS:-$("${GO}" env GOOS)}
 SUFFIX="-${ARCH}"
 GIT_TAG=$DRONE_TAG
 TREE_STATE=clean
@@ -52,6 +53,11 @@ if [ -z "$VERSION_RUNC" ]; then
     VERSION_RUNC="v0.0.0"
 fi
 
+VERSION_HCSSHIM=$(get-module-version github.com/Microsoft/hcsshim)
+if [ -z "$VERSION_HCSSHIM" ]; then
+    VERSION_HCSSHIM="v0.0.0"
+fi
+
 VERSION_FLANNEL=$(get-module-version github.com/flannel-io/flannel)
 if [ -z "$VERSION_FLANNEL" ]; then
   VERSION_FLANNEL="v0.0.0"
@@ -81,3 +87,8 @@ else
     VERSION="$VERSION_K8S+k3s-${COMMIT:0:8}$DIRTY"
 fi
 VERSION_TAG="$(sed -e 's/+/-/g' <<< "$VERSION")"
+
+BINARY_POSTFIX=
+if [ ${OS} = windows ]; then
+    BINARY_POSTFIX=.exe
+fi