Relocate containerd-shim-runsc-v1 deps from shim/ to shim/v1.

BUILD

@@ -133,7 +133,7 @@ go_path(
         # binaries have been factored into a cli package, which is
         # a good practice in any case.
         "//runsc/cli",
-        "//shim/cli",
+        "//shim/v1/cli",
         "//webhook/pkg/cli",
         "//tools/checklocks",
 

nogo.yaml

@@ -84,8 +84,8 @@ global:
       - "panic recovered: .*types/sizes.go:82: assertion failed"
     exclude:
       # Generated: exempt all.
-      - pkg/shim/runtimeoptions/runtimeoptions_cri.go
-      - pkg/shim/runtimeoptions/v14/runtimeoptions_cri.go
+      - pkg/shim/v1/runtimeoptions/runtimeoptions_cri.go
+      - pkg/shim/v1/runtimeoptions/v14/runtimeoptions_cri.go
 analyzers:
   asmdecl:
     generated: # Enabled.

pkg/sentry/control/state.go

@@ -18,8 +18,6 @@ import (
 	"errors"
 	"fmt"
 
-	"gvisor.dev/gvisor/pkg/abi/linux"
-	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
 	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sentry/state"
@@ -84,18 +82,7 @@ func (s *State) Save(o *SaveOpts, _ *struct{}) error {
 		Key:                o.Key,
 		Metadata:           o.Metadata,
 		MemoryFileSaveOpts: o.MemoryFileSaveOpts,
-		Callback: func(err error) {
-			if err == nil {
-				log.Infof("Save succeeded: exiting...")
-				s.Kernel.SetSaveSuccess(false /* autosave */)
-			} else {
-				log.Warningf("Save failed: %v", err)
-				s.Kernel.SetSaveError(err)
-			}
-			if !o.Resume {
-				s.Kernel.Kill(linux.WaitStatusExit(0))
-			}
-		},
+		Resume:             o.Resume,
 	}
 	if o.HavePagesFile {
 		saveOpts.PagesMetadata, err = o.ReleaseFD(1)

pkg/sentry/fsimpl/gofer/save_restore.go

@@ -31,6 +31,8 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 )
 
+var _ vfs.FilesystemImplSaveRestoreExtension = (*filesystem)(nil)
+
 // +stateify savable
 type savedDentryRW struct {
 	read  bool
@@ -134,6 +136,11 @@ func (d *dentry) beforeSave() {
 	}
 }
 
+// BeforeResume implements vfs.FilesystemImplSaveRestoreExtension.BeforeResume.
+func (fs *filesystem) BeforeResume(ctx context.Context) {
+	fs.savedDentryRW = nil
+}
+
 // afterLoad is invoked by stateify.
 func (fs *filesystem) afterLoad(ctx goContext.Context) {
 	fs.mf = pgalloc.MemoryFileFromContext(ctx)

pkg/sentry/fsimpl/tmpfs/save_restore.go

@@ -23,6 +23,8 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 )
 
+var _ vfs.FilesystemImplSaveRestoreExtension = (*filesystem)(nil)
+
 // saveMf is called by stateify.
 func (fs *filesystem) saveMf() string {
 	if !fs.mf.IsSavable() {
@@ -75,6 +77,9 @@ func (fs *filesystem) PrepareSave(ctx context.Context) error {
 	return nil
 }
 
+// BeforeResume implements vfs.FilesystemImplSaveRestoreExtension.BeforeResume.
+func (fs *filesystem) BeforeResume(ctx context.Context) {}
+
 // CompleteRestore implements
 // vfs.FilesystemImplSaveRestoreExtension.CompleteRestore.
 func (fs *filesystem) CompleteRestore(ctx context.Context, opts vfs.CompleteRestoreOptions) error {

pkg/sentry/kernel/kernel.go

@@ -702,6 +702,11 @@ func (k *Kernel) SaveTo(ctx context.Context, w, pagesMetadata io.Writer, pagesFi
 	return nil
 }
 
+// BeforeResume is called before the kernel is resumed after save.
+func (k *Kernel) BeforeResume(ctx context.Context) {
+	k.vfs.BeforeResume(ctx)
+}
+
 func (k *Kernel) saveMemoryFiles(ctx context.Context, w, pagesMetadata io.Writer, pagesFile *fd.FD, mfsToSave map[string]*pgalloc.MemoryFile, mfOpts pgalloc.SaveOpts) error {
 	// Save the memory files' state.
 	memoryStart := time.Now()

pkg/sentry/state/state.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 
+	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
 	"gvisor.dev/gvisor/pkg/fd"
@@ -68,15 +69,18 @@ type SaveOpts struct {
 	// MemoryFileSaveOpts is passed to calls to pgalloc.MemoryFile.SaveTo().
 	MemoryFileSaveOpts pgalloc.SaveOpts
 
-	// Callback is called prior to unpause, with any save error.
-	Callback func(err error)
-
 	// Resume indicates if the statefile is used for save-resume.
 	Resume bool
+
+	// Autosave indicates if the statefile is used for autosave.
+	Autosave bool
 }
 
 // Save saves the system state.
 func (opts SaveOpts) Save(ctx context.Context, k *kernel.Kernel, w *watchdog.Watchdog) error {
+	t, _ := CPUTime()
+	log.Infof("Before save CPU usage: %s", t.String())
+
 	log.Infof("Sandbox save started, pausing all tasks.")
 	k.Pause()
 	k.ReceiveTaskStates()
@@ -127,7 +131,22 @@ func (opts SaveOpts) Save(ctx context.Context, k *kernel.Kernel, w *watchdog.Wat
 			}
 		}
 	}
-	opts.Callback(err)
+
+	t1, _ := CPUTime()
+	log.Infof("Save CPU usage: %s", (t1 - t).String())
+	if err == nil {
+		log.Infof("Save succeeded: exiting...")
+		k.SetSaveSuccess(opts.Autosave)
+	} else {
+		log.Warningf("Save failed: exiting... %v", err)
+		k.SetSaveError(err)
+	}
+	if opts.Resume {
+		k.BeforeResume(ctx)
+	} else {
+		// Kill the sandbox.
+		k.Kill(linux.WaitStatusExit(0))
+	}
 	return err
 }
 

pkg/sentry/vfs/save_restore.go

@@ -54,6 +54,10 @@ type FilesystemImplSaveRestoreExtension interface {
 	// PrepareSave prepares this filesystem for serialization.
 	PrepareSave(ctx context.Context) error
 
+	// BeforeResume is called before the kernel is resumed after save. It can be
+	// used to clean up any state that should be discarded after save.
+	BeforeResume(ctx context.Context)
+
 	// CompleteRestore completes restoration from checkpoint for this
 	// filesystem after deserialization.
 	CompleteRestore(ctx context.Context, opts CompleteRestoreOptions) error
@@ -73,6 +77,17 @@ func (vfs *VirtualFilesystem) PrepareSave(ctx context.Context) error {
 	return nil
 }
 
+// BeforeResume is called before the kernel is resumed after save and allows
+// filesystems to clean up S/R state.
+func (vfs *VirtualFilesystem) BeforeResume(ctx context.Context) {
+	for fs := range vfs.getFilesystems() {
+		if ext, ok := fs.impl.(FilesystemImplSaveRestoreExtension); ok {
+			ext.BeforeResume(ctx)
+		}
+		fs.DecRef(ctx)
+	}
+}
+
 // CompleteRestore completes restoration from checkpoint for all filesystems
 // after deserialization.
 func (vfs *VirtualFilesystem) CompleteRestore(ctx context.Context, opts *CompleteRestoreOptions) error {

pkg/shim/BUILD → pkg/shim/v1/BUILD

@@ -6,13 +6,13 @@ package(
 )
 
 go_library(
-    name = "shim",
+    name = "v1",
     srcs = ["service.go"],
     visibility = ["//shim:__subpackages__"],
     deps = [
         "//pkg/cleanup",
-        "//pkg/shim/extension",
-        "//pkg/shim/runsc",
+        "//pkg/shim/v1/extension",
+        "//pkg/shim/v1/runsc",
         "//pkg/sync",
         "@com_github_containerd_containerd//namespaces:go_default_library",
         "@com_github_containerd_containerd//runtime/v2/shim:go_default_library",

pkg/shim/proc/BUILD → pkg/shim/v1/proc/BUILD

@@ -25,9 +25,9 @@ go_library(
     deps = [
         "//pkg/atomicbitops",
         "//pkg/cleanup",
-        "//pkg/shim/extension",
-        "//pkg/shim/runsccmd",
-        "//pkg/shim/utils",
+        "//pkg/shim/v1/extension",
+        "//pkg/shim/v1/runsccmd",
+        "//pkg/shim/v1/utils",
         "@com_github_containerd_console//:go_default_library",
         "@com_github_containerd_containerd//mount:go_default_library",
         "@com_github_containerd_containerd//pkg/stdio:go_default_library",

pkg/shim/proc/deleted_state.go → pkg/shim/v1/proc/deleted_state.go

@@ -22,7 +22,7 @@ import (
 	"github.com/containerd/console"
 	"github.com/containerd/errdefs"
 	runc "github.com/containerd/go-runc"
-	"gvisor.dev/gvisor/pkg/shim/extension"
+	"gvisor.dev/gvisor/pkg/shim/v1/extension"
 )
 
 type deletedState struct{}

pkg/shim/proc/exec.go → pkg/shim/v1/proc/exec.go

@@ -33,8 +33,8 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/cleanup"
-	"gvisor.dev/gvisor/pkg/shim/extension"
-	"gvisor.dev/gvisor/pkg/shim/runsccmd"
+	"gvisor.dev/gvisor/pkg/shim/v1/extension"
+	"gvisor.dev/gvisor/pkg/shim/v1/runsccmd"
 )
 
 type execProcess struct {

pkg/shim/proc/exec_state.go → pkg/shim/v1/proc/exec_state.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"github.com/containerd/console"
-	"gvisor.dev/gvisor/pkg/shim/extension"
+	"gvisor.dev/gvisor/pkg/shim/v1/extension"
 )
 
 type execState interface {

pkg/shim/proc/init.go → pkg/shim/v1/proc/init.go

@@ -36,9 +36,9 @@ import (
 	runc "github.com/containerd/go-runc"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/sys/unix"
-	"gvisor.dev/gvisor/pkg/shim/extension"
-	"gvisor.dev/gvisor/pkg/shim/runsccmd"
-	"gvisor.dev/gvisor/pkg/shim/utils"
+	"gvisor.dev/gvisor/pkg/shim/v1/extension"
+	"gvisor.dev/gvisor/pkg/shim/v1/runsccmd"
+	"gvisor.dev/gvisor/pkg/shim/v1/utils"
 )
 
 const statusStopped = "stopped"

pkg/shim/proc/init_state.go → pkg/shim/v1/proc/init_state.go

@@ -23,7 +23,7 @@ import (
 	runc "github.com/containerd/go-runc"
 	"golang.org/x/sys/unix"
 
-	"gvisor.dev/gvisor/pkg/shim/extension"
+	"gvisor.dev/gvisor/pkg/shim/v1/extension"
 )
 
 type stateTransition int

pkg/shim/proc/utils.go → pkg/shim/v1/proc/utils.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 	"time"
 
-	"gvisor.dev/gvisor/pkg/shim/runsccmd"
+	"gvisor.dev/gvisor/pkg/shim/v1/runsccmd"
 )
 
 const (

pkg/shim/runsc/BUILD → pkg/shim/v1/runsc/BUILD

@@ -20,17 +20,18 @@ go_library(
     visibility = ["//pkg/shim:__subpackages__"],
     deps = [
         "//pkg/cleanup",
-        "//pkg/shim/extension",
-        "//pkg/shim/proc",
-        "//pkg/shim/runsccmd",
-        "//pkg/shim/runtimeoptions",
-        "//pkg/shim/runtimeoptions/v14",
-        "//pkg/shim/utils",
+        "//pkg/shim/v1/extension",
+        "//pkg/shim/v1/proc",
+        "//pkg/shim/v1/runsccmd",
+        "//pkg/shim/v1/runtimeoptions",
+        "//pkg/shim/v1/runtimeoptions/v14",
+        "//pkg/shim/v1/utils",
         "//runsc/specutils",
         "@com_github_burntsushi_toml//:go_default_library",
         "@com_github_containerd_cgroups//:go_default_library",
         "@com_github_containerd_cgroups//stats/v1:go_default_library",
         "@com_github_containerd_cgroups//v2:go_default_library",
+        "@com_github_containerd_cgroups//v2/stats:go_default_library",
         "@com_github_containerd_console//:go_default_library",
         "@com_github_containerd_containerd//api/events:go_default_library",
         "@com_github_containerd_containerd//api/types/task:go_default_library",
@@ -46,6 +47,7 @@ go_library(
         "@com_github_containerd_containerd//sys/reaper:go_default_library",
         "@com_github_containerd_errdefs//:go_default_library",
         "@com_github_containerd_fifo//:go_default_library",
+        "@com_github_containerd_go_runc//:go_default_library",
         "@com_github_containerd_log//:go_default_library",
         "@com_github_containerd_typeurl//:go_default_library",
         "@com_github_gogo_protobuf//types:go_default_library",
@@ -60,7 +62,7 @@ go_test(
     srcs = ["service_test.go"],
     library = ":runsc",
     deps = [
-        "//pkg/shim/utils",
+        "//pkg/shim/v1/utils",
         "@com_github_opencontainers_runtime_spec//specs-go:go_default_library",
     ],
 )