Resolve #104: Add container support for docker and singularity. (#435)

* Update to latest wtsi-ssg/wr for container RunCmd methods and sec vuln fix.

cmd/add.go

@@ -79,6 +79,9 @@ var cmdFlavor string
 var cmdQueue string
 var cmdMisc string
 var cmdMonitorDocker string
+var cmdWithDocker string
+var cmdWithSingularity string
+var cmdContainerMounts string
 var cmdNoRetry string
 var rtimeoutint int
 var simpleOutput bool
@@ -112,8 +115,9 @@ command as one of the name:value pairs. The possible options are:
 
 cmd cwd cwd_matters change_home on_failure on_success on_exit mounts req_grp
 memory time override cpus disk queue misc priority retries rep_grp dep_grps deps
-cmd_deps monitor_docker cloud_os cloud_username cloud_ram cloud_script
-cloud_config_files cloud_flavor cloud_shared env bsub_mode
+cmd_deps monitor_docker with_docker with_singularity container_mounts cloud_os
+cloud_username cloud_ram cloud_script cloud_config_files cloud_flavor
+cloud_shared env bsub_mode
 
 If any of these will be the same for all your commands, you can instead specify
 them as flags (which are treated as defaults in the case that they are
@@ -314,6 +318,38 @@ NB: does not handle monitoring of multiple docker containers run by a single
 command. A side effect of monitoring a container is that if you use wr to kill
 the job for this command, wr will also kill the container.
 
+"with_docker" takes an image name/location and is a convenience feature that
+will run your command by piping it into 'docker run -i [image] /bin/sh'. Docker
+must therefore be installed on worker nodes for this to work. The image will be
+automatically pulled if it is missing. The container is created with cwd mounted
+and set to the workdir. Any mounts specified by "container_mounts" will also be
+mounted inside the container. Any environment variables you explicitly override
+with "env" will be set inside the container, but not other environment variables
+you have set at the time you add the command. Finally, "monitor_docker" will be
+overridden and set to monitor the container wr creates. If you would need to run
+your docker container with additional options or in a different way, don't use
+"with_docker", and instead have command be your own 'docker run [...]' command,
+and set "monitor_docker" as appropriate.
+
+"with_singularity" takes an image name/location and is a convenience feature
+that will run your command by piping it into 'singularity shell [image]'.
+Singularity must therefore be installed and in the $PATH of worker nodes for
+this to work. The image will be automatically pulled if it is missing. The
+container is created with cwd mounted and set to current directory inside the
+container. Any mounts specified by "container_mounts" will also be mounted
+inside the container. All current and overridden environment variables will be
+set inside the container. If you would need to run your singularity container
+with additional options or in a different way, don't use "with_singularity", and
+instead have command be your own 'singularity [...]' command.
+It's not valid to set both with_docker and with_singularity; if you do, only
+with_docker will be obeyed.
+
+"container_mounts" is a comma separated list of
+/outside/container:/inside/container mount definitions, for use with either
+"with_docker" or "with_singularity". The :/inside path is optional, defaulting
+to the same as the outside path. It will result in the outside paths being
+readable and writable inside the container at the inside paths.
+
 The "cloud_*" related options let you override the defaults of your cloud
 deployment. For example, if you do 'wr cloud deploy --os "Ubuntu 16" --os_ram
 2048 -u ubuntu -s ~/my_ubuntu_post_creation_script.sh', any commands you add
@@ -360,6 +396,10 @@ new job will have this job's mount and cloud_* options.`,
 			die("--file is required")
 		}
 
+		if cmdWithDocker != "" && cmdWithSingularity != "" {
+			die("--with_docker and --with_singularity are mutually exclusive")
+		}
+
 		timeout := time.Duration(timeoutint) * time.Second
 		jq := connect(timeout)
 		var err error
@@ -437,6 +477,9 @@ func init() {
 	addCmd.Flags().StringVar(&cmdCmdDeps, "cmd_deps", "", "dependencies of your commands, in the form \"command1,cwd1,command2,cwd2...\"")
 	addCmd.Flags().StringVarP(&cmdGroupDeps, "deps", "d", "", "dependencies of your commands, in the form \"dep_grp1,dep_grp2...\"")
 	addCmd.Flags().StringVar(&cmdMonitorDocker, "monitor_docker", "", "monitor resource usage of docker container with given --name or --cidfile path")
+	addCmd.Flags().StringVar(&cmdWithDocker, "with_docker", "", "run the cmd inside a docker container running this image")
+	addCmd.Flags().StringVar(&cmdWithSingularity, "with_singularity", "", "run the cmd inside a singularity container running this image")
+	addCmd.Flags().StringVar(&cmdContainerMounts, "container_mounts", "", "mount additional locations inside your container")
 	addCmd.Flags().StringVar(&cmdOnFailure, "on_failure", "", "behaviours to carry out when cmds fails, in JSON format")
 	addCmd.Flags().StringVar(&cmdOnSuccess, "on_success", "", "behaviours to carry out when cmds succeed, in JSON format")
 	addCmd.Flags().StringVar(&cmdOnExit, "on_exit", `[{"cleanup":true}]`, "behaviours to carry out when cmds finish running, in JSON format")
@@ -526,6 +569,9 @@ func parseCmdFile(jq *jobqueue.Client, diskSet bool) ([]*jobqueue.Job, bool, boo
 		Retries:          cmdRet,
 		Env:              cmdEnv,
 		MonitorDocker:    cmdMonitorDocker,
+		WithDocker:       cmdWithDocker,
+		WithSingularity:  cmdWithSingularity,
+		ContainerMounts:  cmdContainerMounts,
 		CloudOS:          cmdOsPrefix,
 		CloudUser:        cmdOsUsername,
 		CloudScript:      cmdPostCreationScript,

cmd/mod.go

@@ -280,6 +280,15 @@ new internal ids is printed.`,
 		if cobraCmd.Flags().Changed("monitor_docker") {
 			jm.SetMonitorDocker(cmdMonitorDocker)
 		}
+		if cobraCmd.Flags().Changed("with_docker") {
+			jm.SetWithDocker(cmdWithDocker)
+		}
+		if cobraCmd.Flags().Changed("with_singularity") {
+			jm.SetWithSingularity(cmdWithSingularity)
+		}
+		if cobraCmd.Flags().Changed("container_mounts") {
+			jm.SetContainerMounts(cmdContainerMounts)
+		}
 
 		var behaviours jobqueue.Behaviours
 		var behavioursSet bool
@@ -386,6 +395,9 @@ func init() {
 	modCmd.Flags().StringVar(&cmdCmdDeps, "cmd_deps", "", "dependencies of your commands, in the form \"command1,cwd1,command2,cwd2...\"")
 	modCmd.Flags().StringVarP(&cmdGroupDeps, "deps", "d", "", "dependencies of your commands, in the form \"dep_grp1,dep_grp2...\"")
 	modCmd.Flags().StringVar(&cmdMonitorDocker, "monitor_docker", "", "monitor resource usage of docker container with given --name or --cidfile path")
+	modCmd.Flags().StringVar(&cmdWithDocker, "with_docker", "", "run the cmd inside a docker container running this image")
+	modCmd.Flags().StringVar(&cmdWithSingularity, "with_singularity", "", "run the cmd inside a singularity container running this image")
+	modCmd.Flags().StringVar(&cmdContainerMounts, "container_mounts", "", "mount additional locations inside your container")
 	modCmd.Flags().StringVar(&cmdOnFailure, "on_failure", "", "behaviours to carry out when cmds fails, in JSON format")
 	modCmd.Flags().StringVar(&cmdOnSuccess, "on_success", "", "behaviours to carry out when cmds succeed, in JSON format")
 	modCmd.Flags().StringVar(&cmdOnExit, "on_exit", `[{"cleanup":true}]`, "behaviours to carry out when cmds finish running, in JSON format")

cmd/status.go

@@ -298,13 +298,22 @@ name to just the first letter, eg. -o c):
 				if len(job.LimitGroups) > 0 {
 					groups += fmt.Sprintf("Limit groups: %s; ", strings.Join(job.LimitGroups, ", "))
 				}
-				var dockerMonitored string
+				var containerInfo string
+				if job.WithDocker != "" {
+					containerInfo = fmt.Sprintf("Cmd running inside docker container running image: %s\n", job.WithDocker)
+				}
+				if job.WithSingularity != "" {
+					containerInfo = fmt.Sprintf("Cmd running inside singularity container running image: %s\n", job.WithSingularity)
+				}
+				if job.ContainerMounts != "" {
+					containerInfo += fmt.Sprintf("Container has these mounts: %s\n", job.ContainerMounts)
+				}
 				if job.MonitorDocker != "" {
 					dockerID := job.MonitorDocker
 					if dockerID == "?" {
 						dockerID += " (first container started after cmd)"
 					}
-					dockerMonitored = fmt.Sprintf("Docker container monitoring turned on for: %s\n", dockerID)
+					containerInfo += fmt.Sprintf("Docker container monitoring turned on for: %s\n", dockerID)
 				}
 				var behaviours string
 				if len(job.Behaviours) > 0 {
@@ -318,7 +327,7 @@ name to just the first letter, eg. -o c):
 					}
 					other = fmt.Sprintf("Resource requirements: %s\n", strings.Join(others, ", "))
 				}
-				fmt.Printf("\n# %s\nCwd: %s\n%s%s%s%s%sId: %s (%s); Requirements group: %s; %sPriority: %d; Attempts: %d\nExpected requirements: { memory: %dMB; time: %s; cpus: %s disk: %dGB }\n", job.Cmd, cwd, mounts, homeChanged, dockerMonitored, behaviours, other, job.RepGroup, job.Key(), job.ReqGroup, groups, job.Priority, job.Attempts, job.Requirements.RAM, job.Requirements.Time, strconv.FormatFloat(job.Requirements.Cores, 'f', -1, 64), job.Requirements.Disk)
+				fmt.Printf("\n# %s\nCwd: %s\n%s%s%s%s%sId: %s (%s); Requirements group: %s; %sPriority: %d; Attempts: %d\nExpected requirements: { memory: %dMB; time: %s; cpus: %s disk: %dGB }\n", job.Cmd, cwd, mounts, homeChanged, containerInfo, behaviours, other, job.RepGroup, job.Key(), job.ReqGroup, groups, job.Priority, job.Attempts, job.Requirements.RAM, job.Requirements.Time, strconv.FormatFloat(job.Requirements.Cores, 'f', -1, 64), job.Requirements.Disk)
 
 				switch job.State {
 				case jobqueue.JobStateDelayed:

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/spf13/cobra v1.2.1
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
 	github.com/ugorji/go/codec v1.2.6
-	github.com/wtsi-ssg/wr v0.4.3
+	github.com/wtsi-ssg/wr v0.5.1
 	go.etcd.io/bbolt v1.3.6
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
@@ -67,8 +67,8 @@ require (
 
 require (
 	github.com/BurntSushi/toml v0.4.1 // indirect
-	github.com/Microsoft/go-winio v0.5.0 // indirect
-	github.com/containerd/containerd v1.5.7 // indirect
+	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/containerd/containerd v1.5.8 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
@@ -94,7 +94,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 // indirect
@@ -104,12 +104,12 @@ require (
 	github.com/smartystreets/assertions v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tklauser/numcpus v0.3.0 // indirect
-	golang.org/x/net v0.0.0-20211006190231-62292e806868 // indirect
-	golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect
+	golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 // indirect
+	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211005153810-c76a74d43a8e // indirect
-	google.golang.org/grpc v1.41.0 // indirect
+	google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect
+	google.golang.org/grpc v1.42.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/ini.v1 v1.63.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect