/* Copyright The containerd Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package runc import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "io/ioutil" "os" "os/exec" "path/filepath" "strconv" "strings" "time" specs "github.com/opencontainers/runtime-spec/specs-go" ) // Format is the type of log formatting options avaliable type Format string // TopBody represents the structured data of the full ps output type TopResults struct { // Processes running in the container, where each is process is an array of values corresponding to the headers Processes [][]string `json:"Processes"` // Headers are the names of the columns Headers []string `json:"Headers"` } const ( none Format = "" JSON Format = "json" Text Format = "text" // DefaultCommand is the default command for Runc DefaultCommand = "runc" ) // List returns all containers created inside the provided runc root directory func (r *Runc) List(context context.Context) ([]*Container, error) { data, err := cmdOutput(r.command(context, "list", "--format=json"), false, nil) defer putBuf(data) if err != nil { return nil, err } var out []*Container if err := json.Unmarshal(data.Bytes(), &out); err != nil { return nil, err } return out, nil } // State returns the state for the container provided by id func (r *Runc) State(context context.Context, id string) (*Container, error) { data, err := cmdOutput(r.command(context, "state", id), true, nil) defer putBuf(data) if err != nil { return nil, fmt.Errorf("%s: %s", err, data.String()) } var c Container if err := json.Unmarshal(data.Bytes(), &c); err != nil { return nil, err } return &c, nil } type ConsoleSocket interface { Path() string } type CreateOpts struct { IO // PidFile is a path to where a pid file should be created PidFile string ConsoleSocket ConsoleSocket Detach bool NoPivot bool NoNewKeyring bool ExtraFiles []*os.File Started chan<- int } func (o *CreateOpts) args() (out []string, err error) { if o.PidFile != "" { abs, err := filepath.Abs(o.PidFile) if err != nil { return nil, err } out = append(out, "--pid-file", abs) } if o.ConsoleSocket != nil { out = append(out, "--console-socket", o.ConsoleSocket.Path()) } if o.NoPivot { out = append(out, "--no-pivot") } if o.NoNewKeyring { out = append(out, "--no-new-keyring") } if o.Detach { out = append(out, "--detach") } if o.ExtraFiles != nil { out = append(out, "--preserve-fds", strconv.Itoa(len(o.ExtraFiles))) } return out, nil } // Create creates a new container and returns its pid if it was created successfully func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error { args := []string{"create", "--bundle", bundle} if opts != nil { oargs, err := opts.args() if err != nil { return err } args = append(args, oargs...) } cmd := r.command(context, append(args, id)...) if opts != nil && opts.IO != nil { opts.Set(cmd) } cmd.ExtraFiles = opts.ExtraFiles if cmd.Stdout == nil && cmd.Stderr == nil { data, err := cmdOutput(cmd, true, nil) defer putBuf(data) if err != nil { return fmt.Errorf("%s: %s", err, data.String()) } return nil } ec, err := Monitor.Start(cmd) if err != nil { return err } if opts != nil && opts.IO != nil { if c, ok := opts.IO.(StartCloser); ok { if err := c.CloseAfterStart(); err != nil { return err } } } status, err := Monitor.Wait(cmd, ec) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate successfully: %w", cmd.Args[0], &ExitError{status}) } return err } // Start will start an already created container func (r *Runc) Start(context context.Context, id string) error { return r.runOrError(r.command(context, "start", id)) } type ExecOpts struct { IO PidFile string ConsoleSocket ConsoleSocket Detach bool Started chan<- int } func (o *ExecOpts) args() (out []string, err error) { if o.ConsoleSocket != nil { out = append(out, "--console-socket", o.ConsoleSocket.Path()) } if o.Detach { out = append(out, "--detach") } if o.PidFile != "" { abs, err := filepath.Abs(o.PidFile) if err != nil { return nil, err } out = append(out, "--pid-file", abs) } return out, nil } // Exec executes an additional process inside the container based on a full // OCI Process specification func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error { if opts.Started != nil { defer close(opts.Started) } f, err := ioutil.TempFile(os.Getenv("XDG_RUNTIME_DIR"), "runc-process") if err != nil { return err } defer os.Remove(f.Name()) err = json.NewEncoder(f).Encode(spec) f.Close() if err != nil { return err } args := []string{"exec", "--process", f.Name()} if opts != nil { oargs, err := opts.args() if err != nil { return err } args = append(args, oargs...) } cmd := r.command(context, append(args, id)...) if opts != nil && opts.IO != nil { opts.Set(cmd) } if cmd.Stdout == nil && cmd.Stderr == nil { data, err := cmdOutput(cmd, true, opts.Started) defer putBuf(data) if err != nil { return fmt.Errorf("%w: %s", err, data.String()) } return nil } ec, err := Monitor.Start(cmd) if err != nil { return err } if opts.Started != nil { opts.Started <- cmd.Process.Pid } if opts != nil && opts.IO != nil { if c, ok := opts.IO.(StartCloser); ok { if err := c.CloseAfterStart(); err != nil { return err } } } status, err := Monitor.Wait(cmd, ec) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate successfully: %w", cmd.Args[0], &ExitError{status}) } return err } // Run runs the create, start, delete lifecycle of the container // and returns its exit status after it has exited func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) (int, error) { if opts.Started != nil { defer close(opts.Started) } args := []string{"run", "--bundle", bundle} if opts != nil { oargs, err := opts.args() if err != nil { return -1, err } args = append(args, oargs...) } cmd := r.command(context, append(args, id)...) if opts != nil && opts.IO != nil { opts.Set(cmd) } ec, err := Monitor.Start(cmd) if err != nil { return -1, err } if opts.Started != nil { opts.Started <- cmd.Process.Pid } status, err := Monitor.Wait(cmd, ec) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate successfully: %w", cmd.Args[0], &ExitError{status}) } return status, err } type DeleteOpts struct { Force bool } func (o *DeleteOpts) args() (out []string) { if o.Force { out = append(out, "--force") } return out } // Delete deletes the container func (r *Runc) Delete(context context.Context, id string, opts *DeleteOpts) error { args := []string{"delete"} if opts != nil { args = append(args, opts.args()...) } return r.runOrError(r.command(context, append(args, id)...)) } // KillOpts specifies options for killing a container and its processes type KillOpts struct { All bool } func (o *KillOpts) args() (out []string) { if o.All { out = append(out, "--all") } return out } // Kill sends the specified signal to the container func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) error { args := []string{ "kill", } if opts != nil { args = append(args, opts.args()...) } return r.runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...)) } // Stats return the stats for a container like cpu, memory, and io func (r *Runc) Stats(context context.Context, id string) (*Stats, error) { cmd := r.command(context, "events", "--stats", id) rd, err := cmd.StdoutPipe() if err != nil { return nil, err } ec, err := Monitor.Start(cmd) if err != nil { return nil, err } defer func() { rd.Close() Monitor.Wait(cmd, ec) }() var e Event if err := json.NewDecoder(rd).Decode(&e); err != nil { return nil, err } return e.Stats, nil } // Events returns an event stream from runc for a container with stats and OOM notifications func (r *Runc) Events(context context.Context, id string, interval time.Duration) (chan *Event, error) { cmd := r.command(context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id) rd, err := cmd.StdoutPipe() if err != nil { return nil, err } ec, err := Monitor.Start(cmd) if err != nil { rd.Close() return nil, err } var ( dec = json.NewDecoder(rd) c = make(chan *Event, 128) ) go func() { defer func() { close(c) rd.Close() Monitor.Wait(cmd, ec) }() for { var e Event if err := dec.Decode(&e); err != nil { if err == io.EOF { return } e = Event{ Type: "error", Err: err, } } c <- &e } }() return c, nil } // Pause the container with the provided id func (r *Runc) Pause(context context.Context, id string) error { return r.runOrError(r.command(context, "pause", id)) } // Resume the container with the provided id func (r *Runc) Resume(context context.Context, id string) error { return r.runOrError(r.command(context, "resume", id)) } // Ps lists all the processes inside the container returning their pids func (r *Runc) Ps(context context.Context, id string) ([]int, error) { data, err := cmdOutput(r.command(context, "ps", "--format", "json", id), true, nil) defer putBuf(data) if err != nil { return nil, fmt.Errorf("%s: %s", err, data.String()) } var pids []int if err := json.Unmarshal(data.Bytes(), &pids); err != nil { return nil, err } return pids, nil } // Top lists all the processes inside the container returning the full ps data func (r *Runc) Top(context context.Context, id string, psOptions string) (*TopResults, error) { data, err := cmdOutput(r.command(context, "ps", "--format", "table", id, psOptions), true, nil) defer putBuf(data) if err != nil { return nil, fmt.Errorf("%s: %s", err, data.String()) } topResults, err := ParsePSOutput(data.Bytes()) if err != nil { return nil, fmt.Errorf("%s: ", err) } return topResults, nil } type CheckpointOpts struct { // ImagePath is the path for saving the criu image file ImagePath string // WorkDir is the working directory for criu WorkDir string // ParentPath is the path for previous image files from a pre-dump ParentPath string // AllowOpenTCP allows open tcp connections to be checkpointed AllowOpenTCP bool // AllowExternalUnixSockets allows external unix sockets to be checkpointed AllowExternalUnixSockets bool // AllowTerminal allows the terminal(pty) to be checkpointed with a container AllowTerminal bool // CriuPageServer is the address:port for the criu page server CriuPageServer string // FileLocks handle file locks held by the container FileLocks bool // Cgroups is the cgroup mode for how to handle the checkpoint of a container's cgroups Cgroups CgroupMode // EmptyNamespaces creates a namespace for the container but does not save its properties // Provide the namespaces you wish to be checkpointed without their settings on restore EmptyNamespaces []string // LazyPages uses userfaultfd to lazily restore memory pages LazyPages bool // StatusFile is the file criu writes \0 to once lazy-pages is ready StatusFile *os.File } type CgroupMode string const ( Soft CgroupMode = "soft" Full CgroupMode = "full" Strict CgroupMode = "strict" ) func (o *CheckpointOpts) args() (out []string) { if o.ImagePath != "" { out = append(out, "--image-path", o.ImagePath) } if o.WorkDir != "" { out = append(out, "--work-path", o.WorkDir) } if o.ParentPath != "" { out = append(out, "--parent-path", o.ParentPath) } if o.AllowOpenTCP { out = append(out, "--tcp-established") } if o.AllowExternalUnixSockets { out = append(out, "--ext-unix-sk") } if o.AllowTerminal { out = append(out, "--shell-job") } if o.CriuPageServer != "" { out = append(out, "--page-server", o.CriuPageServer) } if o.FileLocks { out = append(out, "--file-locks") } if string(o.Cgroups) != "" { out = append(out, "--manage-cgroups-mode", string(o.Cgroups)) } for _, ns := range o.EmptyNamespaces { out = append(out, "--empty-ns", ns) } if o.LazyPages { out = append(out, "--lazy-pages") } return out } type CheckpointAction func([]string) []string // LeaveRunning keeps the container running after the checkpoint has been completed func LeaveRunning(args []string) []string { return append(args, "--leave-running") } // PreDump allows a pre-dump of the checkpoint to be made and completed later func PreDump(args []string) []string { return append(args, "--pre-dump") } // Checkpoint allows you to checkpoint a container using criu func (r *Runc) Checkpoint(context context.Context, id string, opts *CheckpointOpts, actions ...CheckpointAction) error { args := []string{"checkpoint"} extraFiles := []*os.File{} if opts != nil { args = append(args, opts.args()...) if opts.StatusFile != nil { // pass the status file to the child process extraFiles = []*os.File{opts.StatusFile} // set status-fd to 3 as this will be the file descriptor // of the first file passed with cmd.ExtraFiles args = append(args, "--status-fd", "3") } } for _, a := range actions { args = a(args) } cmd := r.command(context, append(args, id)...) cmd.ExtraFiles = extraFiles return r.runOrError(cmd) } type RestoreOpts struct { CheckpointOpts IO Detach bool PidFile string NoSubreaper bool NoPivot bool ConsoleSocket ConsoleSocket } func (o *RestoreOpts) args() ([]string, error) { out := o.CheckpointOpts.args() if o.Detach { out = append(out, "--detach") } if o.PidFile != "" { abs, err := filepath.Abs(o.PidFile) if err != nil { return nil, err } out = append(out, "--pid-file", abs) } if o.ConsoleSocket != nil { out = append(out, "--console-socket", o.ConsoleSocket.Path()) } if o.NoPivot { out = append(out, "--no-pivot") } if o.NoSubreaper { out = append(out, "-no-subreaper") } return out, nil } // Restore restores a container with the provide id from an existing checkpoint func (r *Runc) Restore(context context.Context, id, bundle string, opts *RestoreOpts) (int, error) { args := []string{"restore"} if opts != nil { oargs, err := opts.args() if err != nil { return -1, err } args = append(args, oargs...) } args = append(args, "--bundle", bundle) cmd := r.command(context, append(args, id)...) if opts != nil && opts.IO != nil { opts.Set(cmd) } ec, err := Monitor.Start(cmd) if err != nil { return -1, err } if opts != nil && opts.IO != nil { if c, ok := opts.IO.(StartCloser); ok { if err := c.CloseAfterStart(); err != nil { return -1, err } } } status, err := Monitor.Wait(cmd, ec) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate successfully: %w", cmd.Args[0], &ExitError{status}) } return status, err } // Update updates the current container with the provided resource spec func (r *Runc) Update(context context.Context, id string, resources *specs.LinuxResources) error { buf := getBuf() defer putBuf(buf) if err := json.NewEncoder(buf).Encode(resources); err != nil { return err } args := []string{"update", "--resources", "-", id} cmd := r.command(context, args...) cmd.Stdin = buf return r.runOrError(cmd) } var ErrParseRuncVersion = errors.New("unable to parse runc version") type Version struct { Runc string Commit string Spec string } // Version returns the runc and runtime-spec versions func (r *Runc) Version(context context.Context) (Version, error) { data, err := cmdOutput(r.command(context, "--version"), false, nil) defer putBuf(data) if err != nil { return Version{}, err } return parseVersion(data.Bytes()) } func parseVersion(data []byte) (Version, error) { var v Version parts := strings.Split(strings.TrimSpace(string(data)), "\n") if len(parts) > 0 { if !strings.HasPrefix(parts[0], "runc version ") { return v, nil } v.Runc = parts[0][13:] for _, part := range parts[1:] { if strings.HasPrefix(part, "commit: ") { v.Commit = part[8:] } else if strings.HasPrefix(part, "spec: ") { v.Spec = part[6:] } } } return v, nil } func (r *Runc) args() (out []string) { if r.Root != "" { out = append(out, "--root", r.Root) } if r.Debug { out = append(out, "--debug") } if r.Log != "" { out = append(out, "--log", r.Log) } if r.LogFormat != none { out = append(out, "--log-format", string(r.LogFormat)) } if r.Criu != "" { out = append(out, "--criu", r.Criu) } if r.SystemdCgroup { out = append(out, "--systemd-cgroup") } if r.Rootless != nil { // nil stands for "auto" (differs from explicit "false") out = append(out, "--rootless="+strconv.FormatBool(*r.Rootless)) } return out } // runOrError will run the provided command. If an error is // encountered and neither Stdout or Stderr was set the error and the // stderr of the command will be returned in the format of : // func (r *Runc) runOrError(cmd *exec.Cmd) error { if cmd.Stdout != nil || cmd.Stderr != nil { ec, err := Monitor.Start(cmd) if err != nil { return err } status, err := Monitor.Wait(cmd, ec) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate successfully: %w", cmd.Args[0], &ExitError{status}) } return err } data, err := cmdOutput(cmd, true, nil) defer putBuf(data) if err != nil { return fmt.Errorf("%s: %s", err, data.String()) } return nil } // callers of cmdOutput are expected to call putBuf on the returned Buffer // to ensure it is released back to the shared pool after use. func cmdOutput(cmd *exec.Cmd, combined bool, started chan<- int) (*bytes.Buffer, error) { b := getBuf() cmd.Stdout = b if combined { cmd.Stderr = b } ec, err := Monitor.Start(cmd) if err != nil { return nil, err } if started != nil { started <- cmd.Process.Pid } status, err := Monitor.Wait(cmd, ec) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate successfully: %w", cmd.Args[0], &ExitError{status}) } return b, err } type ExitError struct { Status int } func (e *ExitError) Error() string { return fmt.Sprintf("exit status %d", e.Status) }