mirror of
https://github.com/k3s-io/k3s.git
synced 2024-06-07 19:41:36 +00:00
984 lines
29 KiB
Go
984 lines
29 KiB
Go
// +build linux
|
|
|
|
package libcontainer
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
securejoin "github.com/cyphar/filepath-securejoin"
|
|
"github.com/moby/sys/mountinfo"
|
|
"github.com/mrunalp/fileutils"
|
|
"github.com/opencontainers/runc/libcontainer/cgroups"
|
|
"github.com/opencontainers/runc/libcontainer/configs"
|
|
"github.com/opencontainers/runc/libcontainer/system"
|
|
libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
|
|
"github.com/opencontainers/runtime-spec/specs-go"
|
|
"github.com/opencontainers/selinux/go-selinux/label"
|
|
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
const defaultMountFlags = unix.MS_NOEXEC | unix.MS_NOSUID | unix.MS_NODEV
|
|
|
|
// needsSetupDev returns true if /dev needs to be set up.
|
|
func needsSetupDev(config *configs.Config) bool {
|
|
for _, m := range config.Mounts {
|
|
if m.Device == "bind" && libcontainerUtils.CleanPath(m.Destination) == "/dev" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// prepareRootfs sets up the devices, mount points, and filesystems for use
|
|
// inside a new mount namespace. It doesn't set anything as ro. You must call
|
|
// finalizeRootfs after this function to finish setting up the rootfs.
|
|
func prepareRootfs(pipe io.ReadWriter, iConfig *initConfig) (err error) {
|
|
config := iConfig.Config
|
|
if err := prepareRoot(config); err != nil {
|
|
return newSystemErrorWithCause(err, "preparing rootfs")
|
|
}
|
|
|
|
hasCgroupns := config.Namespaces.Contains(configs.NEWCGROUP)
|
|
setupDev := needsSetupDev(config)
|
|
for _, m := range config.Mounts {
|
|
for _, precmd := range m.PremountCmds {
|
|
if err := mountCmd(precmd); err != nil {
|
|
return newSystemErrorWithCause(err, "running premount command")
|
|
}
|
|
}
|
|
if err := mountToRootfs(m, config.Rootfs, config.MountLabel, hasCgroupns); err != nil {
|
|
return newSystemErrorWithCausef(err, "mounting %q to rootfs at %q", m.Source, m.Destination)
|
|
}
|
|
|
|
for _, postcmd := range m.PostmountCmds {
|
|
if err := mountCmd(postcmd); err != nil {
|
|
return newSystemErrorWithCause(err, "running postmount command")
|
|
}
|
|
}
|
|
}
|
|
|
|
if setupDev {
|
|
if err := createDevices(config); err != nil {
|
|
return newSystemErrorWithCause(err, "creating device nodes")
|
|
}
|
|
if err := setupPtmx(config); err != nil {
|
|
return newSystemErrorWithCause(err, "setting up ptmx")
|
|
}
|
|
if err := setupDevSymlinks(config.Rootfs); err != nil {
|
|
return newSystemErrorWithCause(err, "setting up /dev symlinks")
|
|
}
|
|
}
|
|
|
|
// Signal the parent to run the pre-start hooks.
|
|
// The hooks are run after the mounts are setup, but before we switch to the new
|
|
// root, so that the old root is still available in the hooks for any mount
|
|
// manipulations.
|
|
// Note that iConfig.Cwd is not guaranteed to exist here.
|
|
if err := syncParentHooks(pipe); err != nil {
|
|
return err
|
|
}
|
|
|
|
// The reason these operations are done here rather than in finalizeRootfs
|
|
// is because the console-handling code gets quite sticky if we have to set
|
|
// up the console before doing the pivot_root(2). This is because the
|
|
// Console API has to also work with the ExecIn case, which means that the
|
|
// API must be able to deal with being inside as well as outside the
|
|
// container. It's just cleaner to do this here (at the expense of the
|
|
// operation not being perfectly split).
|
|
|
|
if err := unix.Chdir(config.Rootfs); err != nil {
|
|
return newSystemErrorWithCausef(err, "changing dir to %q", config.Rootfs)
|
|
}
|
|
|
|
s := iConfig.SpecState
|
|
s.Pid = unix.Getpid()
|
|
s.Status = specs.StateCreating
|
|
if err := iConfig.Config.Hooks[configs.CreateContainer].RunHooks(s); err != nil {
|
|
return err
|
|
}
|
|
|
|
if config.NoPivotRoot {
|
|
err = msMoveRoot(config.Rootfs)
|
|
} else if config.Namespaces.Contains(configs.NEWNS) {
|
|
err = pivotRoot(config.Rootfs)
|
|
} else {
|
|
err = chroot()
|
|
}
|
|
if err != nil {
|
|
return newSystemErrorWithCause(err, "jailing process inside rootfs")
|
|
}
|
|
|
|
if setupDev {
|
|
if err := reOpenDevNull(); err != nil {
|
|
return newSystemErrorWithCause(err, "reopening /dev/null inside container")
|
|
}
|
|
}
|
|
|
|
if cwd := iConfig.Cwd; cwd != "" {
|
|
// Note that spec.Process.Cwd can contain unclean value like "../../../../foo/bar...".
|
|
// However, we are safe to call MkDirAll directly because we are in the jail here.
|
|
if err := os.MkdirAll(cwd, 0755); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// finalizeRootfs sets anything to ro if necessary. You must call
|
|
// prepareRootfs first.
|
|
func finalizeRootfs(config *configs.Config) (err error) {
|
|
// remount dev as ro if specified
|
|
for _, m := range config.Mounts {
|
|
if libcontainerUtils.CleanPath(m.Destination) == "/dev" {
|
|
if m.Flags&unix.MS_RDONLY == unix.MS_RDONLY {
|
|
if err := remountReadonly(m); err != nil {
|
|
return newSystemErrorWithCausef(err, "remounting %q as readonly", m.Destination)
|
|
}
|
|
}
|
|
break
|
|
}
|
|
}
|
|
|
|
// set rootfs ( / ) as readonly
|
|
if config.Readonlyfs {
|
|
if err := setReadonly(); err != nil {
|
|
return newSystemErrorWithCause(err, "setting rootfs as readonly")
|
|
}
|
|
}
|
|
|
|
unix.Umask(0022)
|
|
return nil
|
|
}
|
|
|
|
// /tmp has to be mounted as private to allow MS_MOVE to work in all situations
|
|
func prepareTmp(topTmpDir string) (string, error) {
|
|
tmpdir, err := ioutil.TempDir(topTmpDir, "runctop")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err := unix.Mount(tmpdir, tmpdir, "bind", unix.MS_BIND, ""); err != nil {
|
|
return "", err
|
|
}
|
|
if err := unix.Mount("", tmpdir, "", uintptr(unix.MS_PRIVATE), ""); err != nil {
|
|
return "", err
|
|
}
|
|
return tmpdir, nil
|
|
}
|
|
|
|
func cleanupTmp(tmpdir string) error {
|
|
unix.Unmount(tmpdir, 0)
|
|
return os.RemoveAll(tmpdir)
|
|
}
|
|
|
|
func mountCmd(cmd configs.Command) error {
|
|
command := exec.Command(cmd.Path, cmd.Args[:]...)
|
|
command.Env = cmd.Env
|
|
command.Dir = cmd.Dir
|
|
if out, err := command.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("%#v failed: %s: %v", cmd, string(out), err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func prepareBindMount(m *configs.Mount, rootfs string) error {
|
|
stat, err := os.Stat(m.Source)
|
|
if err != nil {
|
|
// error out if the source of a bind mount does not exist as we will be
|
|
// unable to bind anything to it.
|
|
return err
|
|
}
|
|
// ensure that the destination of the bind mount is resolved of symlinks at mount time because
|
|
// any previous mounts can invalidate the next mount's destination.
|
|
// this can happen when a user specifies mounts within other mounts to cause breakouts or other
|
|
// evil stuff to try to escape the container's rootfs.
|
|
var dest string
|
|
if dest, err = securejoin.SecureJoin(rootfs, m.Destination); err != nil {
|
|
return err
|
|
}
|
|
if err := checkProcMount(rootfs, dest, m.Source); err != nil {
|
|
return err
|
|
}
|
|
// update the mount with the correct dest after symlinks are resolved.
|
|
m.Destination = dest
|
|
if err := createIfNotExists(dest, stat.IsDir()); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func mountCgroupV1(m *configs.Mount, rootfs, mountLabel string, enableCgroupns bool) error {
|
|
binds, err := getCgroupMounts(m)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var merged []string
|
|
for _, b := range binds {
|
|
ss := filepath.Base(b.Destination)
|
|
if strings.Contains(ss, ",") {
|
|
merged = append(merged, ss)
|
|
}
|
|
}
|
|
tmpfs := &configs.Mount{
|
|
Source: "tmpfs",
|
|
Device: "tmpfs",
|
|
Destination: m.Destination,
|
|
Flags: defaultMountFlags,
|
|
Data: "mode=755",
|
|
PropagationFlags: m.PropagationFlags,
|
|
}
|
|
if err := mountToRootfs(tmpfs, rootfs, mountLabel, enableCgroupns); err != nil {
|
|
return err
|
|
}
|
|
for _, b := range binds {
|
|
if enableCgroupns {
|
|
subsystemPath := filepath.Join(rootfs, b.Destination)
|
|
if err := os.MkdirAll(subsystemPath, 0755); err != nil {
|
|
return err
|
|
}
|
|
flags := defaultMountFlags
|
|
if m.Flags&unix.MS_RDONLY != 0 {
|
|
flags = flags | unix.MS_RDONLY
|
|
}
|
|
cgroupmount := &configs.Mount{
|
|
Source: "cgroup",
|
|
Device: "cgroup", // this is actually fstype
|
|
Destination: subsystemPath,
|
|
Flags: flags,
|
|
Data: filepath.Base(subsystemPath),
|
|
}
|
|
if err := mountNewCgroup(cgroupmount); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
if err := mountToRootfs(b, rootfs, mountLabel, enableCgroupns); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
for _, mc := range merged {
|
|
for _, ss := range strings.Split(mc, ",") {
|
|
// symlink(2) is very dumb, it will just shove the path into
|
|
// the link and doesn't do any checks or relative path
|
|
// conversion. Also, don't error out if the cgroup already exists.
|
|
if err := os.Symlink(mc, filepath.Join(rootfs, m.Destination, ss)); err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func mountCgroupV2(m *configs.Mount, rootfs, mountLabel string, enableCgroupns bool) error {
|
|
cgroupPath, err := securejoin.SecureJoin(rootfs, m.Destination)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(cgroupPath, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := unix.Mount(m.Source, cgroupPath, "cgroup2", uintptr(m.Flags), m.Data); err != nil {
|
|
// when we are in UserNS but CgroupNS is not unshared, we cannot mount cgroup2 (#2158)
|
|
if err == unix.EPERM || err == unix.EBUSY {
|
|
return unix.Mount("/sys/fs/cgroup", cgroupPath, "", uintptr(m.Flags)|unix.MS_BIND, "")
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func mountToRootfs(m *configs.Mount, rootfs, mountLabel string, enableCgroupns bool) error {
|
|
var (
|
|
dest = m.Destination
|
|
)
|
|
if !strings.HasPrefix(dest, rootfs) {
|
|
dest = filepath.Join(rootfs, dest)
|
|
}
|
|
|
|
switch m.Device {
|
|
case "proc", "sysfs":
|
|
// If the destination already exists and is not a directory, we bail
|
|
// out This is to avoid mounting through a symlink or similar -- which
|
|
// has been a "fun" attack scenario in the past.
|
|
// TODO: This won't be necessary once we switch to libpathrs and we can
|
|
// stop all of these symlink-exchange attacks.
|
|
if fi, err := os.Lstat(dest); err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
} else if fi.Mode()&os.ModeDir == 0 {
|
|
return fmt.Errorf("filesystem %q must be mounted on ordinary directory", m.Device)
|
|
}
|
|
if err := os.MkdirAll(dest, 0755); err != nil {
|
|
return err
|
|
}
|
|
// Selinux kernels do not support labeling of /proc or /sys
|
|
return mountPropagate(m, rootfs, "")
|
|
case "mqueue":
|
|
if err := os.MkdirAll(dest, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := mountPropagate(m, rootfs, mountLabel); err != nil {
|
|
// older kernels do not support labeling of /dev/mqueue
|
|
if err := mountPropagate(m, rootfs, ""); err != nil {
|
|
return err
|
|
}
|
|
return label.SetFileLabel(dest, mountLabel)
|
|
}
|
|
return nil
|
|
case "tmpfs":
|
|
copyUp := m.Extensions&configs.EXT_COPYUP == configs.EXT_COPYUP
|
|
tmpDir := ""
|
|
stat, err := os.Stat(dest)
|
|
if err != nil {
|
|
if err := os.MkdirAll(dest, 0755); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if copyUp {
|
|
tmpdir, err := prepareTmp("/tmp")
|
|
if err != nil {
|
|
return newSystemErrorWithCause(err, "tmpcopyup: failed to setup tmpdir")
|
|
}
|
|
defer cleanupTmp(tmpdir)
|
|
tmpDir, err = ioutil.TempDir(tmpdir, "runctmpdir")
|
|
if err != nil {
|
|
return newSystemErrorWithCause(err, "tmpcopyup: failed to create tmpdir")
|
|
}
|
|
defer os.RemoveAll(tmpDir)
|
|
m.Destination = tmpDir
|
|
}
|
|
if err := mountPropagate(m, rootfs, mountLabel); err != nil {
|
|
return err
|
|
}
|
|
if copyUp {
|
|
if err := fileutils.CopyDirectory(dest, tmpDir); err != nil {
|
|
errMsg := fmt.Errorf("tmpcopyup: failed to copy %s to %s: %v", dest, tmpDir, err)
|
|
if err1 := unix.Unmount(tmpDir, unix.MNT_DETACH); err1 != nil {
|
|
return newSystemErrorWithCausef(err1, "tmpcopyup: %v: failed to unmount", errMsg)
|
|
}
|
|
return errMsg
|
|
}
|
|
if err := unix.Mount(tmpDir, dest, "", unix.MS_MOVE, ""); err != nil {
|
|
errMsg := fmt.Errorf("tmpcopyup: failed to move mount %s to %s: %v", tmpDir, dest, err)
|
|
if err1 := unix.Unmount(tmpDir, unix.MNT_DETACH); err1 != nil {
|
|
return newSystemErrorWithCausef(err1, "tmpcopyup: %v: failed to unmount", errMsg)
|
|
}
|
|
return errMsg
|
|
}
|
|
}
|
|
if stat != nil {
|
|
if err = os.Chmod(dest, stat.Mode()); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
case "bind":
|
|
if err := prepareBindMount(m, rootfs); err != nil {
|
|
return err
|
|
}
|
|
if err := mountPropagate(m, rootfs, mountLabel); err != nil {
|
|
return err
|
|
}
|
|
// bind mount won't change mount options, we need remount to make mount options effective.
|
|
// first check that we have non-default options required before attempting a remount
|
|
if m.Flags&^(unix.MS_REC|unix.MS_REMOUNT|unix.MS_BIND) != 0 {
|
|
// only remount if unique mount options are set
|
|
if err := remount(m, rootfs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if m.Relabel != "" {
|
|
if err := label.Validate(m.Relabel); err != nil {
|
|
return err
|
|
}
|
|
shared := label.IsShared(m.Relabel)
|
|
if err := label.Relabel(m.Source, mountLabel, shared); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
case "cgroup":
|
|
if cgroups.IsCgroup2UnifiedMode() {
|
|
return mountCgroupV2(m, rootfs, mountLabel, enableCgroupns)
|
|
}
|
|
return mountCgroupV1(m, rootfs, mountLabel, enableCgroupns)
|
|
default:
|
|
// ensure that the destination of the mount is resolved of symlinks at mount time because
|
|
// any previous mounts can invalidate the next mount's destination.
|
|
// this can happen when a user specifies mounts within other mounts to cause breakouts or other
|
|
// evil stuff to try to escape the container's rootfs.
|
|
var err error
|
|
if dest, err = securejoin.SecureJoin(rootfs, m.Destination); err != nil {
|
|
return err
|
|
}
|
|
if err := checkProcMount(rootfs, dest, m.Source); err != nil {
|
|
return err
|
|
}
|
|
// update the mount with the correct dest after symlinks are resolved.
|
|
m.Destination = dest
|
|
if err := os.MkdirAll(dest, 0755); err != nil {
|
|
return err
|
|
}
|
|
return mountPropagate(m, rootfs, mountLabel)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func getCgroupMounts(m *configs.Mount) ([]*configs.Mount, error) {
|
|
mounts, err := cgroups.GetCgroupMounts(false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cgroupPaths, err := cgroups.ParseCgroupFile("/proc/self/cgroup")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var binds []*configs.Mount
|
|
|
|
for _, mm := range mounts {
|
|
dir, err := mm.GetOwnCgroup(cgroupPaths)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
relDir, err := filepath.Rel(mm.Root, dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
binds = append(binds, &configs.Mount{
|
|
Device: "bind",
|
|
Source: filepath.Join(mm.Mountpoint, relDir),
|
|
Destination: filepath.Join(m.Destination, filepath.Base(mm.Mountpoint)),
|
|
Flags: unix.MS_BIND | unix.MS_REC | m.Flags,
|
|
PropagationFlags: m.PropagationFlags,
|
|
})
|
|
}
|
|
|
|
return binds, nil
|
|
}
|
|
|
|
// checkProcMount checks to ensure that the mount destination is not over the top of /proc.
|
|
// dest is required to be an abs path and have any symlinks resolved before calling this function.
|
|
//
|
|
// if source is nil, don't stat the filesystem. This is used for restore of a checkpoint.
|
|
func checkProcMount(rootfs, dest, source string) error {
|
|
const procPath = "/proc"
|
|
// White list, it should be sub directories of invalid destinations
|
|
validDestinations := []string{
|
|
// These entries can be bind mounted by files emulated by fuse,
|
|
// so commands like top, free displays stats in container.
|
|
"/proc/cpuinfo",
|
|
"/proc/diskstats",
|
|
"/proc/meminfo",
|
|
"/proc/stat",
|
|
"/proc/swaps",
|
|
"/proc/uptime",
|
|
"/proc/loadavg",
|
|
"/proc/net/dev",
|
|
}
|
|
for _, valid := range validDestinations {
|
|
path, err := filepath.Rel(filepath.Join(rootfs, valid), dest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if path == "." {
|
|
return nil
|
|
}
|
|
}
|
|
path, err := filepath.Rel(filepath.Join(rootfs, procPath), dest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// pass if the mount path is located outside of /proc
|
|
if strings.HasPrefix(path, "..") {
|
|
return nil
|
|
}
|
|
if path == "." {
|
|
// an empty source is pasted on restore
|
|
if source == "" {
|
|
return nil
|
|
}
|
|
// only allow a mount on-top of proc if it's source is "proc"
|
|
isproc, err := isProc(source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// pass if the mount is happening on top of /proc and the source of
|
|
// the mount is a proc filesystem
|
|
if isproc {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%q cannot be mounted because it is not of type proc", dest)
|
|
}
|
|
return fmt.Errorf("%q cannot be mounted because it is inside /proc", dest)
|
|
}
|
|
|
|
func isProc(path string) (bool, error) {
|
|
var s unix.Statfs_t
|
|
if err := unix.Statfs(path, &s); err != nil {
|
|
return false, err
|
|
}
|
|
return s.Type == unix.PROC_SUPER_MAGIC, nil
|
|
}
|
|
|
|
func setupDevSymlinks(rootfs string) error {
|
|
var links = [][2]string{
|
|
{"/proc/self/fd", "/dev/fd"},
|
|
{"/proc/self/fd/0", "/dev/stdin"},
|
|
{"/proc/self/fd/1", "/dev/stdout"},
|
|
{"/proc/self/fd/2", "/dev/stderr"},
|
|
}
|
|
// kcore support can be toggled with CONFIG_PROC_KCORE; only create a symlink
|
|
// in /dev if it exists in /proc.
|
|
if _, err := os.Stat("/proc/kcore"); err == nil {
|
|
links = append(links, [2]string{"/proc/kcore", "/dev/core"})
|
|
}
|
|
for _, link := range links {
|
|
var (
|
|
src = link[0]
|
|
dst = filepath.Join(rootfs, link[1])
|
|
)
|
|
if err := os.Symlink(src, dst); err != nil && !os.IsExist(err) {
|
|
return fmt.Errorf("symlink %s %s %s", src, dst, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// If stdin, stdout, and/or stderr are pointing to `/dev/null` in the parent's rootfs
|
|
// this method will make them point to `/dev/null` in this container's rootfs. This
|
|
// needs to be called after we chroot/pivot into the container's rootfs so that any
|
|
// symlinks are resolved locally.
|
|
func reOpenDevNull() error {
|
|
var stat, devNullStat unix.Stat_t
|
|
file, err := os.OpenFile("/dev/null", os.O_RDWR, 0)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to open /dev/null - %s", err)
|
|
}
|
|
defer file.Close()
|
|
if err := unix.Fstat(int(file.Fd()), &devNullStat); err != nil {
|
|
return err
|
|
}
|
|
for fd := 0; fd < 3; fd++ {
|
|
if err := unix.Fstat(fd, &stat); err != nil {
|
|
return err
|
|
}
|
|
if stat.Rdev == devNullStat.Rdev {
|
|
// Close and re-open the fd.
|
|
if err := unix.Dup3(int(file.Fd()), fd, 0); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Create the device nodes in the container.
|
|
func createDevices(config *configs.Config) error {
|
|
useBindMount := system.RunningInUserNS() || config.Namespaces.Contains(configs.NEWUSER)
|
|
oldMask := unix.Umask(0000)
|
|
for _, node := range config.Devices {
|
|
// containers running in a user namespace are not allowed to mknod
|
|
// devices so we can just bind mount it from the host.
|
|
if err := createDeviceNode(config.Rootfs, node, useBindMount); err != nil {
|
|
unix.Umask(oldMask)
|
|
return err
|
|
}
|
|
}
|
|
unix.Umask(oldMask)
|
|
return nil
|
|
}
|
|
|
|
func bindMountDeviceNode(dest string, node *configs.Device) error {
|
|
f, err := os.Create(dest)
|
|
if err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
if f != nil {
|
|
f.Close()
|
|
}
|
|
return unix.Mount(node.Path, dest, "bind", unix.MS_BIND, "")
|
|
}
|
|
|
|
// Creates the device node in the rootfs of the container.
|
|
func createDeviceNode(rootfs string, node *configs.Device, bind bool) error {
|
|
if node.Path == "" {
|
|
// The node only exists for cgroup reasons, ignore it here.
|
|
return nil
|
|
}
|
|
dest := filepath.Join(rootfs, node.Path)
|
|
if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
|
|
return err
|
|
}
|
|
if bind {
|
|
return bindMountDeviceNode(dest, node)
|
|
}
|
|
if err := mknodDevice(dest, node); err != nil {
|
|
if os.IsExist(err) {
|
|
return nil
|
|
} else if os.IsPermission(err) {
|
|
return bindMountDeviceNode(dest, node)
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func mknodDevice(dest string, node *configs.Device) error {
|
|
fileMode := node.FileMode
|
|
switch node.Type {
|
|
case configs.BlockDevice:
|
|
fileMode |= unix.S_IFBLK
|
|
case configs.CharDevice:
|
|
fileMode |= unix.S_IFCHR
|
|
case configs.FifoDevice:
|
|
fileMode |= unix.S_IFIFO
|
|
default:
|
|
return fmt.Errorf("%c is not a valid device type for device %s", node.Type, node.Path)
|
|
}
|
|
dev, err := node.Mkdev()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := unix.Mknod(dest, uint32(fileMode), int(dev)); err != nil {
|
|
return err
|
|
}
|
|
return unix.Chown(dest, int(node.Uid), int(node.Gid))
|
|
}
|
|
|
|
// Get the parent mount point of directory passed in as argument. Also return
|
|
// optional fields.
|
|
func getParentMount(rootfs string) (string, string, error) {
|
|
mi, err := mountinfo.GetMounts(mountinfo.ParentsFilter(rootfs))
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if len(mi) < 1 {
|
|
return "", "", fmt.Errorf("could not find parent mount of %s", rootfs)
|
|
}
|
|
|
|
// find the longest mount point
|
|
var idx, maxlen int
|
|
for i := range mi {
|
|
if len(mi[i].Mountpoint) > maxlen {
|
|
maxlen = len(mi[i].Mountpoint)
|
|
idx = i
|
|
}
|
|
}
|
|
return mi[idx].Mountpoint, mi[idx].Optional, nil
|
|
}
|
|
|
|
// Make parent mount private if it was shared
|
|
func rootfsParentMountPrivate(rootfs string) error {
|
|
sharedMount := false
|
|
|
|
parentMount, optionalOpts, err := getParentMount(rootfs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
optsSplit := strings.Split(optionalOpts, " ")
|
|
for _, opt := range optsSplit {
|
|
if strings.HasPrefix(opt, "shared:") {
|
|
sharedMount = true
|
|
break
|
|
}
|
|
}
|
|
|
|
// Make parent mount PRIVATE if it was shared. It is needed for two
|
|
// reasons. First of all pivot_root() will fail if parent mount is
|
|
// shared. Secondly when we bind mount rootfs it will propagate to
|
|
// parent namespace and we don't want that to happen.
|
|
if sharedMount {
|
|
return unix.Mount("", parentMount, "", unix.MS_PRIVATE, "")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func prepareRoot(config *configs.Config) error {
|
|
flag := unix.MS_SLAVE | unix.MS_REC
|
|
if config.RootPropagation != 0 {
|
|
flag = config.RootPropagation
|
|
}
|
|
if err := unix.Mount("", "/", "", uintptr(flag), ""); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Make parent mount private to make sure following bind mount does
|
|
// not propagate in other namespaces. Also it will help with kernel
|
|
// check pass in pivot_root. (IS_SHARED(new_mnt->mnt_parent))
|
|
if err := rootfsParentMountPrivate(config.Rootfs); err != nil {
|
|
return err
|
|
}
|
|
|
|
return unix.Mount(config.Rootfs, config.Rootfs, "bind", unix.MS_BIND|unix.MS_REC, "")
|
|
}
|
|
|
|
func setReadonly() error {
|
|
return unix.Mount("/", "/", "bind", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_REC, "")
|
|
}
|
|
|
|
func setupPtmx(config *configs.Config) error {
|
|
ptmx := filepath.Join(config.Rootfs, "dev/ptmx")
|
|
if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
if err := os.Symlink("pts/ptmx", ptmx); err != nil {
|
|
return fmt.Errorf("symlink dev ptmx %s", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// pivotRoot will call pivot_root such that rootfs becomes the new root
|
|
// filesystem, and everything else is cleaned up.
|
|
func pivotRoot(rootfs string) error {
|
|
// While the documentation may claim otherwise, pivot_root(".", ".") is
|
|
// actually valid. What this results in is / being the new root but
|
|
// /proc/self/cwd being the old root. Since we can play around with the cwd
|
|
// with pivot_root this allows us to pivot without creating directories in
|
|
// the rootfs. Shout-outs to the LXC developers for giving us this idea.
|
|
|
|
oldroot, err := unix.Open("/", unix.O_DIRECTORY|unix.O_RDONLY, 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer unix.Close(oldroot)
|
|
|
|
newroot, err := unix.Open(rootfs, unix.O_DIRECTORY|unix.O_RDONLY, 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer unix.Close(newroot)
|
|
|
|
// Change to the new root so that the pivot_root actually acts on it.
|
|
if err := unix.Fchdir(newroot); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := unix.PivotRoot(".", "."); err != nil {
|
|
return fmt.Errorf("pivot_root %s", err)
|
|
}
|
|
|
|
// Currently our "." is oldroot (according to the current kernel code).
|
|
// However, purely for safety, we will fchdir(oldroot) since there isn't
|
|
// really any guarantee from the kernel what /proc/self/cwd will be after a
|
|
// pivot_root(2).
|
|
|
|
if err := unix.Fchdir(oldroot); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Make oldroot rslave to make sure our unmounts don't propagate to the
|
|
// host (and thus bork the machine). We don't use rprivate because this is
|
|
// known to cause issues due to races where we still have a reference to a
|
|
// mount while a process in the host namespace are trying to operate on
|
|
// something they think has no mounts (devicemapper in particular).
|
|
if err := unix.Mount("", ".", "", unix.MS_SLAVE|unix.MS_REC, ""); err != nil {
|
|
return err
|
|
}
|
|
// Preform the unmount. MNT_DETACH allows us to unmount /proc/self/cwd.
|
|
if err := unix.Unmount(".", unix.MNT_DETACH); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Switch back to our shiny new root.
|
|
if err := unix.Chdir("/"); err != nil {
|
|
return fmt.Errorf("chdir / %s", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func msMoveRoot(rootfs string) error {
|
|
mountinfos, err := mountinfo.GetMounts(func(info *mountinfo.Info) (skip, stop bool) {
|
|
skip = false
|
|
stop = false
|
|
// Collect every sysfs and proc file systems, except those under the container rootfs
|
|
if (info.Fstype != "proc" && info.Fstype != "sysfs") || strings.HasPrefix(info.Mountpoint, rootfs) {
|
|
skip = true
|
|
return
|
|
}
|
|
return
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, info := range mountinfos {
|
|
p := info.Mountpoint
|
|
// Be sure umount events are not propagated to the host.
|
|
if err := unix.Mount("", p, "", unix.MS_SLAVE|unix.MS_REC, ""); err != nil {
|
|
return err
|
|
}
|
|
if err := unix.Unmount(p, unix.MNT_DETACH); err != nil {
|
|
if err != unix.EINVAL && err != unix.EPERM {
|
|
return err
|
|
} else {
|
|
// If we have not privileges for umounting (e.g. rootless), then
|
|
// cover the path.
|
|
if err := unix.Mount("tmpfs", p, "tmpfs", 0, ""); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err := unix.Mount(rootfs, "/", "", unix.MS_MOVE, ""); err != nil {
|
|
return err
|
|
}
|
|
return chroot()
|
|
}
|
|
|
|
func chroot() error {
|
|
if err := unix.Chroot("."); err != nil {
|
|
return err
|
|
}
|
|
return unix.Chdir("/")
|
|
}
|
|
|
|
// createIfNotExists creates a file or a directory only if it does not already exist.
|
|
func createIfNotExists(path string, isDir bool) error {
|
|
if _, err := os.Stat(path); err != nil {
|
|
if os.IsNotExist(err) {
|
|
if isDir {
|
|
return os.MkdirAll(path, 0755)
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
|
return err
|
|
}
|
|
f, err := os.OpenFile(path, os.O_CREATE, 0755)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f.Close()
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// readonlyPath will make a path read only.
|
|
func readonlyPath(path string) error {
|
|
if err := unix.Mount(path, path, "", unix.MS_BIND|unix.MS_REC, ""); err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
return unix.Mount(path, path, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_REC, "")
|
|
}
|
|
|
|
// remountReadonly will remount an existing mount point and ensure that it is read-only.
|
|
func remountReadonly(m *configs.Mount) error {
|
|
var (
|
|
dest = m.Destination
|
|
flags = m.Flags
|
|
)
|
|
for i := 0; i < 5; i++ {
|
|
// There is a special case in the kernel for
|
|
// MS_REMOUNT | MS_BIND, which allows us to change only the
|
|
// flags even as an unprivileged user (i.e. user namespace)
|
|
// assuming we don't drop any security related flags (nodev,
|
|
// nosuid, etc.). So, let's use that case so that we can do
|
|
// this re-mount without failing in a userns.
|
|
flags |= unix.MS_REMOUNT | unix.MS_BIND | unix.MS_RDONLY
|
|
if err := unix.Mount("", dest, "", uintptr(flags), ""); err != nil {
|
|
switch err {
|
|
case unix.EBUSY:
|
|
time.Sleep(100 * time.Millisecond)
|
|
continue
|
|
default:
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf("unable to mount %s as readonly max retries reached", dest)
|
|
}
|
|
|
|
// maskPath masks the top of the specified path inside a container to avoid
|
|
// security issues from processes reading information from non-namespace aware
|
|
// mounts ( proc/kcore ).
|
|
// For files, maskPath bind mounts /dev/null over the top of the specified path.
|
|
// For directories, maskPath mounts read-only tmpfs over the top of the specified path.
|
|
func maskPath(path string, mountLabel string) error {
|
|
if err := unix.Mount("/dev/null", path, "", unix.MS_BIND, ""); err != nil && !os.IsNotExist(err) {
|
|
if err == unix.ENOTDIR {
|
|
return unix.Mount("tmpfs", path, "tmpfs", unix.MS_RDONLY, label.FormatMountLabel("", mountLabel))
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
|
|
// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward.
|
|
func writeSystemProperty(key, value string) error {
|
|
keyPath := strings.Replace(key, ".", "/", -1)
|
|
return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
|
|
}
|
|
|
|
func remount(m *configs.Mount, rootfs string) error {
|
|
var (
|
|
dest = m.Destination
|
|
)
|
|
if !strings.HasPrefix(dest, rootfs) {
|
|
dest = filepath.Join(rootfs, dest)
|
|
}
|
|
return unix.Mount(m.Source, dest, m.Device, uintptr(m.Flags|unix.MS_REMOUNT), "")
|
|
}
|
|
|
|
// Do the mount operation followed by additional mounts required to take care
|
|
// of propagation flags.
|
|
func mountPropagate(m *configs.Mount, rootfs string, mountLabel string) error {
|
|
var (
|
|
dest = m.Destination
|
|
data = label.FormatMountLabel(m.Data, mountLabel)
|
|
flags = m.Flags
|
|
)
|
|
if libcontainerUtils.CleanPath(dest) == "/dev" {
|
|
flags &= ^unix.MS_RDONLY
|
|
}
|
|
|
|
copyUp := m.Extensions&configs.EXT_COPYUP == configs.EXT_COPYUP
|
|
if !(copyUp || strings.HasPrefix(dest, rootfs)) {
|
|
dest = filepath.Join(rootfs, dest)
|
|
}
|
|
|
|
if err := unix.Mount(m.Source, dest, m.Device, uintptr(flags), data); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, pflag := range m.PropagationFlags {
|
|
if err := unix.Mount("", dest, "", uintptr(pflag), ""); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func mountNewCgroup(m *configs.Mount) error {
|
|
var (
|
|
data = m.Data
|
|
source = m.Source
|
|
)
|
|
if data == "systemd" {
|
|
data = cgroups.CgroupNamePrefix + data
|
|
source = "systemd"
|
|
}
|
|
if err := unix.Mount(source, m.Destination, m.Device, uintptr(m.Flags), data); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|