Unshare Pea limits from Container

Move Pea namespace sharing out of Bundler [#151709987] Signed-off-by: Will Martin <[email protected]>
hpcloud · Oct 30, 2017 · a0d37e2 · a0d37e2
1 parent ba009ed
commit a0d37e2
Show file tree

Hide file tree

Showing 19 changed files with 294 additions and 116 deletions.
diff --git a/gardener/gardener.go b/gardener/gardener.go
@@ -129,6 +129,10 @@ func (fn UidGeneratorFunc) Generate() string {
 type DesiredContainerSpec struct {
 	Handle string
 
+	CgroupPath string
+
+	Namespaces map[string]string
+
 	// Container hostname
 	Hostname string
 

diff --git a/guardiancmd/command.go b/guardiancmd/command.go
@@ -34,6 +34,7 @@ import (
 	"code.cloudfoundry.org/guardian/rundmc/cgroups"
 	"code.cloudfoundry.org/guardian/rundmc/goci"
 	"code.cloudfoundry.org/guardian/rundmc/peas"
+	"code.cloudfoundry.org/guardian/rundmc/pidreader"
 	"code.cloudfoundry.org/guardian/rundmc/preparerootfs"
 	"code.cloudfoundry.org/guardian/rundmc/runrunc"
 	"code.cloudfoundry.org/guardian/rundmc/stopper"
@@ -743,6 +744,10 @@ func (cmd *ServerCommand) wireContainerizer(log lager.Logger,
 			ContainerRootGID: gidMappings.Map(0),
 			MkdirChown:       chrootMkdir,
 		},
+		bundlerules.Namespaces{},
+		bundlerules.CGroupPath{
+			Path: cgroupRootPath,
+		},
 		bundlerules.Mounts{},
 		bundlerules.Env{},
 		bundlerules.Hostname{},
@@ -752,14 +757,11 @@ func (cmd *ServerCommand) wireContainerizer(log lager.Logger,
 	peaBundleRules := make([]rundmc.BundlerRule, len(bundleRules))
 	copy(peaBundleRules, bundleRules)
 	bundleRules = append(bundleRules,
-		bundlerules.CGroupPath{
-			Path: cgroupRootPath,
-		}, bundlerules.Limits{
+		bundlerules.Limits{
 			CpuQuotaPerShare: cmd.Limits.CPUQuotaPerShare,
 			TCPMemoryLimit:   int64(cmd.Limits.TCPMemoryLimit),
 			BlockIOWeight:    cmd.Limits.DefaultBlockIOWeight,
 		})
-	peaBundleRules = append(peaBundleRules, bundlerules.NamespaceSharing{})
 
 	template := &rundmc.BundleTemplate{Rules: bundleRules}
 	peaTemplate := &rundmc.BundleTemplate{Rules: peaBundleRules}
@@ -801,15 +803,23 @@ func (cmd *ServerCommand) wireContainerizer(log lager.Logger,
 		}
 	}
 
-	nstar := rundmc.NewNstarRunner(nstarPath, tarPath, cmdRunner)
-	stopper := stopper.New(stopper.NewRuncStateCgroupPathResolver(runcRoot), nil, retrier.New(retrier.ConstantBackoff(10, 1*time.Second), nil))
+	pidFileReader := &pidreader.PidFileReader{
+		Clock:         clock.NewClock(),
+		Timeout:       10 * time.Second,
+		SleepInterval: time.Millisecond * 100,
+	}
+
 	peaCreator := &peas.PeaCreator{
 		VolumeCreator:    volumeCreator,
+		PidGetter:        pidFileReader,
 		BundleGenerator:  peaTemplate,
 		ProcessBuilder:   processBuilder,
 		BundleSaver:      bundleSaver,
 		ContainerCreator: runrunc.NewCreator(runtimePath, "run", runtimeExtraArgs, cmdRunner),
 	}
+
+	nstar := rundmc.NewNstarRunner(nstarPath, tarPath, cmdRunner)
+	stopper := stopper.New(stopper.NewRuncStateCgroupPathResolver(runcRoot), nil, retrier.New(retrier.ConstantBackoff(10, 1*time.Second), nil))
 	return rundmc.New(depot, runcrunner, bndlLoader, nstar, stopper, eventStore, stateStore, &preparerootfs.SymlinkRefusingFileCreator{}, peaCreator)
 }
 

diff --git a/guardiancmd/command_linux.go b/guardiancmd/command_linux.go
@@ -24,6 +24,7 @@ import (
 	"code.cloudfoundry.org/guardian/rundmc/cgroups"
 	"code.cloudfoundry.org/guardian/rundmc/depot"
 	"code.cloudfoundry.org/guardian/rundmc/execrunner/dadoo"
+	"code.cloudfoundry.org/guardian/rundmc/pidreader"
 	"code.cloudfoundry.org/guardian/rundmc/preparerootfs"
 	"code.cloudfoundry.org/guardian/rundmc/runrunc"
 	"code.cloudfoundry.org/idmapper"
@@ -182,8 +183,7 @@ func (NoopMkdirer) MkdirAs(rootFSPathFile string, uid, gid int, mode os.FileMode
 }
 
 func (cmd *ServerCommand) wireExecRunner(dadooPath, runcPath string, processIDGen runrunc.UidGenerator, commandRunner commandrunner.CommandRunner, shouldCleanup bool) *dadoo.ExecRunner {
-
-	pidFileReader := &dadoo.PidFileReader{
+	pidFileReader := &pidreader.PidFileReader{
 		Clock:         clock.NewClock(),
 		Timeout:       10 * time.Second,
 		SleepInterval: time.Millisecond * 100,

diff --git a/rundmc/bundlerules/base.go b/rundmc/bundlerules/base.go
@@ -11,7 +11,7 @@ type Base struct {
 	UnprivilegedBase goci.Bndl
 }
 
-func (r Base) Apply(bndl goci.Bndl, spec gardener.DesiredContainerSpec, containerDir string) (goci.Bndl, error) {
+func (r Base) Apply(bndl goci.Bndl, spec gardener.DesiredContainerSpec, _ string) (goci.Bndl, error) {
 	if spec.Privileged {
 		copiedBndl, err := copystructure.Copy(r.PrivilegedBase)
 		if err != nil {

diff --git a/rundmc/bundlerules/cgroup_path.go b/rundmc/bundlerules/cgroup_path.go
@@ -16,5 +16,9 @@ func (r CGroupPath) Apply(bndl goci.Bndl, spec gardener.DesiredContainerSpec, _
 		return bndl, nil
 	}
 
+	if spec.CgroupPath != "" {
+		return bndl.WithCGroupPath(filepath.Join(r.Path, spec.CgroupPath)), nil
+	}
+
 	return bndl.WithCGroupPath(filepath.Join(r.Path, spec.Handle)), nil
 }
diff --git a/rundmc/bundlerules/mounts.go b/rundmc/bundlerules/mounts.go
@@ -10,7 +10,7 @@ import (
 type Mounts struct {
 }
 
-func (b Mounts) Apply(bndl goci.Bndl, spec gardener.DesiredContainerSpec, containerDir string) (goci.Bndl, error) {
+func (b Mounts) Apply(bndl goci.Bndl, spec gardener.DesiredContainerSpec, _ string) (goci.Bndl, error) {
 	var mounts []specs.Mount
 	for _, m := range spec.BindMounts {
 		modeOpt := "ro"

diff --git a/rundmc/bundlerules/namespace_sharing.go b/rundmc/bundlerules/namespace_sharing.go
diff --git a/rundmc/bundlerules/namespace_sharing_test.go b/rundmc/bundlerules/namespace_sharing_test.go
diff --git a/rundmc/bundlerules/namespaces.go b/rundmc/bundlerules/namespaces.go
@@ -0,0 +1,17 @@
+package bundlerules
+
+import (
+	"code.cloudfoundry.org/guardian/gardener"
+	"code.cloudfoundry.org/guardian/rundmc/goci"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type Namespaces struct{}
+
+func (n Namespaces) Apply(bndl goci.Bndl, spec gardener.DesiredContainerSpec, containerDir string) (goci.Bndl, error) {
+	for ns, path := range spec.Namespaces {
+		bndl = bndl.WithNamespace(specs.LinuxNamespace{Type: specs.LinuxNamespaceType(ns), Path: path})
+	}
+
+	return bndl, nil
+}
diff --git a/rundmc/bundlerules/namespaces_test.go b/rundmc/bundlerules/namespaces_test.go
@@ -0,0 +1,26 @@
+package bundlerules_test
+
+import (
+	"code.cloudfoundry.org/guardian/gardener"
+	"code.cloudfoundry.org/guardian/rundmc/bundlerules"
+	"code.cloudfoundry.org/guardian/rundmc/goci"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+var _ = Describe("Namespaces", func() {
+	It("sets all namespaces in the bundle to those provided by the spec", func() {
+		initialBndl := goci.Bundle()
+
+		desiredContainerSpec := gardener.DesiredContainerSpec{Namespaces: map[string]string{"mount": "", "network": "test-net-ns", "user": "test-user-ns"}}
+		transformedBndl, err := bundlerules.Namespaces{}.Apply(initialBndl, desiredContainerSpec, "")
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(transformedBndl.Namespaces()).To(ConsistOf(
+			specs.LinuxNamespace{Type: "mount"},
+			specs.LinuxNamespace{Type: "network", Path: "test-net-ns"},
+			specs.LinuxNamespace{Type: "user", Path: "test-user-ns"},
+		))
+	})
+})
diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go
@@ -48,7 +48,7 @@ type OCIRuntime interface {
 }
 
 type PeaCreator interface {
-	CreatePea(log lager.Logger, spec garden.ProcessSpec, pio garden.ProcessIO, ctrBundlePath string) (garden.Process, error)
+	CreatePea(log lager.Logger, spec garden.ProcessSpec, pio garden.ProcessIO, ctrHandle, ctrBundlePath string) (garden.Process, error)
 }
 
 type NstarRunner interface {
@@ -152,7 +152,7 @@ func (c *Containerizer) Run(log lager.Logger, handle string, spec garden.Process
 	}
 
 	if spec.Image != (garden.ImageRef{}) {
-		return c.peaCreator.CreatePea(log, spec, io, path)
+		return c.peaCreator.CreatePea(log, spec, io, handle, path)
 	}
 
 	return c.runtime.Exec(log, path, handle, spec, io)

diff --git a/rundmc/containerizer_test.go b/rundmc/containerizer_test.go
@@ -184,8 +184,9 @@ var _ = Describe("Rundmc", func() {
 				processSpec := garden.ProcessSpec{Image: garden.ImageRef{URI: "some-uri"}}
 				containerizer.Run(logger, "some-handle", processSpec, pio)
 				Expect(fakePeaCreator.CreatePeaCallCount()).To(Equal(1))
-				_, actualProcessSpec, actualProcessIO, actualBundlePath := fakePeaCreator.CreatePeaArgsForCall(0)
+				_, actualProcessSpec, actualProcessIO, actualHandle, actualBundlePath := fakePeaCreator.CreatePeaArgsForCall(0)
 				Expect(actualProcessSpec).To(Equal(processSpec))
+				Expect(actualHandle).To(Equal("some-handle"))
 				Expect(actualBundlePath).To(Equal("some-bundle-path"))
 				Expect(actualProcessIO).To(Equal(pio))
 			})

diff --git a/rundmc/peas/pea_creator.go b/rundmc/peas/pea_creator.go
@@ -32,15 +32,21 @@ type VolumeCreator interface {
 	Create(log lager.Logger, spec garden.ContainerSpec) (specs.Spec, error)
 }
 
+//go:generate counterfeiter . PidGetter
+type PidGetter interface {
+	Pid(pidFilePath string) (int, error)
+}
+
 type PeaCreator struct {
 	VolumeCreator    VolumeCreator
+	PidGetter        PidGetter
 	BundleGenerator  depot.BundleGenerator
 	BundleSaver      depot.BundleSaver
 	ProcessBuilder   runrunc.ProcessBuilder
 	ContainerCreator ContainerCreator
 }
 
-func (p *PeaCreator) CreatePea(log lager.Logger, spec garden.ProcessSpec, procIO garden.ProcessIO, ctrBundlePath string) (garden.Process, error) {
+func (p *PeaCreator) CreatePea(log lager.Logger, spec garden.ProcessSpec, procIO garden.ProcessIO, ctrHandle, ctrBundlePath string) (garden.Process, error) {
 	errs := func(action string, err error) (garden.Process, error) {
 		wrappedErr := errorwrapper.Wrap(err, action)
 		log.Error(action, wrappedErr)
@@ -70,9 +76,29 @@ func (p *PeaCreator) CreatePea(log lager.Logger, spec garden.ProcessSpec, procIO
 		return errs("creating-volume", err)
 	}
 
+	cgroupPath := ctrHandle
+	if spec.OverrideContainerLimits != nil {
+		cgroupPath = processID
+	}
+
+	originalCtrInitPid, err := p.PidGetter.Pid(filepath.Join(ctrBundlePath, "pidfile"))
+	if err != nil {
+		return errs("reading-ctr-pid", err)
+	}
+
+	linuxNamespaces := map[string]string{}
+	linuxNamespaces["mount"] = ""
+	linuxNamespaces["network"] = fmt.Sprintf("/proc/%d/ns/net", originalCtrInitPid)
+	linuxNamespaces["user"] = fmt.Sprintf("/proc/%d/ns/user", originalCtrInitPid)
+	linuxNamespaces["ipc"] = fmt.Sprintf("/proc/%d/ns/ipc", originalCtrInitPid)
+	linuxNamespaces["pid"] = fmt.Sprintf("/proc/%d/ns/pid", originalCtrInitPid)
+	linuxNamespaces["uts"] = fmt.Sprintf("/proc/%d/ns/uts", originalCtrInitPid)
+
 	bndl, err := p.BundleGenerator.Generate(gardener.DesiredContainerSpec{
 		Handle:     processID,
 		BaseConfig: runtimeSpec,
+		CgroupPath: cgroupPath,
+		Namespaces: linuxNamespaces,
 	}, ctrBundlePath)
 	if err != nil {
 		return errs("generating-bundle", err)