Merge branch 'release/0.18.1'

.travis.yml

@@ -13,7 +13,7 @@ env:
     - WR_MANAGERPORT="11301"
     - GO111MODULE=on
 go:
-  - "1.11.5"
+  - "1.12.5"
 go_import_path: github.com/VertebrateResequencing/wr
 install:
   - "go mod verify"

.vscode/settings.json

@@ -3,7 +3,7 @@
   "editor.formatOnSave": true,
   "go.lintTool": "gometalinter",
   "go.vetOnSave": "workspace",
-  "go.formatTool": "goreturns",
+  "go.formatTool": "goimports",
   "go.lintFlags": [
     "--vendor",
     "--disable-all",

CHANGELOG.md

@@ -5,6 +5,25 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this
 project adheres to [Semantic Versioning](http://semver.org/).
 
 
+## [0.18.1] - 2019-05-23
+### Added
+- New `--manager_flavor` option to `wr cloud deploy` (defaulting to new
+  cloudflavormanager config option), to be able to set a different flavor for
+  the deployed manager compared to workers created by the manager.
+
+### Changed
+- At least go v1.12 is needed to compile wr.
+- cloudos config option (and --os option to `wr cloud deploy` or --cloud_os
+  option to `wr manager start`) has always picked a random image that had a
+  matching prefix; now an image with an exactly matching name will be picked in
+  preference.
+- Default cloudos changed from "Ubuntu Xenial" to "bionic-server".
+
+### Fixed
+- Fixed a situation where OpenStack servers failed to scale down and became
+  unused when adding lots of quickly completing jobs.
+
+
 ## [0.18.0] - 2019-04-08
 ### Added
 - New limit group property on Jobs, to allow you to limit the number of jobs

README.md

@@ -44,15 +44,15 @@ Download
 --------
 [![download](https://img.shields.io/badge/download-wr-green.svg)](https://github.com/VertebrateResequencing/wr/releases)
 
-Alternatively, build it yourself (at least v1.11.5 of go is required):
+Alternatively, build it yourself (at least v1.12 of go is required):
 
 1. Install go on your machine according to:
 [golang.org/doc/install](https://golang.org/doc/install)
 An example way of setting up a personal Go installation in your home directory
 would be:
 
-        wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz
-        tar -xvzf go1.11.5.linux-amd64.tar.gz && rm go1.11.5.linux-amd64.tar.gz
+        wget https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz
+        tar -xvzf go1.12.5.linux-amd64.tar.gz && rm go1.12.5.linux-amd64.tar.gz
         export PATH=$PATH:$HOME/go/bin
 
 2. Download, compile, and install wr (not inside $GOPATH, if you set that):

cloud/openstack.go

@@ -277,6 +277,7 @@ func (p *openstackp) cacheImages() error {
 			if i.Progress == 100 {
 				thisI := i // copy before storing ref
 				p.imap[i.ID] = &thisI
+				p.imap[i.Name] = &thisI
 			}
 		}
 
@@ -310,6 +311,13 @@ func (p *openstackp) getImage(prefix string) (*images.Image, error) {
 func (p *openstackp) getImageFromCache(prefix string) *images.Image {
 	p.imapMutex.RLock()
 	defer p.imapMutex.RUnlock()
+
+	// find an exact match
+	if i, found := p.imap[prefix]; found {
+		return i
+	}
+
+	// failing that, find a random prefix match
 	for _, i := range p.imap {
 		if strings.HasPrefix(i.Name, prefix) || strings.HasPrefix(i.ID, prefix) {
 			return i

cmd/cloud.go

@@ -61,6 +61,7 @@ var osUsername string
 var osRAM int
 var osDisk int
 var flavorRegex string
+var managerFlavor string
 var flavorSets string
 var postCreationScript string
 var postDeploymentScript string
@@ -342,7 +343,10 @@ within OpenStack.`,
 		}
 		if server == nil {
 			info("please wait while a server is spawned on %s...", providerName)
-			flavor, errf := provider.CheapestServerFlavor(1, osRAM, flavorRegex)
+			if managerFlavor == "" {
+				managerFlavor = flavorRegex
+			}
+			flavor, errf := provider.CheapestServerFlavor(1, osRAM, managerFlavor)
 			if errf != nil {
 				teardown(provider)
 				die("failed to launch a server in %s: %s", providerName, errf)
@@ -717,6 +721,11 @@ func init() {
 	cloudDeployCmd.Flags().IntVarP(&osRAM, "os_ram", "r", defaultConfig.CloudRAM, "ram (MB) needed by the OS image specified by --os")
 	cloudDeployCmd.Flags().IntVarP(&osDisk, "os_disk", "d", defaultConfig.CloudDisk, "minimum disk (GB) for servers")
 	cloudDeployCmd.Flags().StringVarP(&flavorRegex, "flavor", "f", defaultConfig.CloudFlavor, "a regular expression to limit server flavors that can be automatically picked")
+	defaultNote := ""
+	if defaultConfig.CloudFlavorManager == "" {
+		defaultNote = " (defaults to --flavor if blank)"
+	}
+	cloudDeployCmd.Flags().StringVar(&managerFlavor, "manager_flavor", defaultConfig.CloudFlavorManager, "like --flavor, but specific to the first server created to run the manager"+defaultNote)
 	cloudDeployCmd.Flags().StringVar(&flavorSets, "flavor_sets", defaultConfig.CloudFlavorSets, "sets of flavors assigned to different hardware, in the form f1,f2;f3,f4")
 	cloudDeployCmd.Flags().StringVarP(&postCreationScript, "script", "s", defaultConfig.CloudScript, "path to a start-up script that will be run on each server created")
 	cloudDeployCmd.Flags().IntVar(&maxManagerCores, "max_local_cores", -1, "maximum number of manager cores to use to run cmds; -1 means unlimited")

cmd/kubernetes.go

@@ -776,7 +776,10 @@ func init() {
 // gets preserved across containers.
 func rewriteConfigFiles(configFiles string) []client.FilePair {
 	// Get current user's home directory
-	hDir := os.Getenv("HOME")
+	hDir, herr := os.UserHomeDir()
+	if herr != nil {
+		warn("could not find home dir: %s", herr)
+	}
 
 	filePairs := []client.FilePair{}
 	paths := []string{}

go.mod

@@ -1,98 +1,140 @@
 module github.com/VertebrateResequencing/wr
 
 require (
-	code.cloudfoundry.org/bytefmt v0.0.0-20180108190415-b31f603f5e1e
-	github.com/BurntSushi/toml v0.3.1 // indirect
-	github.com/Microsoft/go-winio v0.4.11 // indirect
-	github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
+	cloud.google.com/go v0.39.0 // indirect
+	code.cloudfoundry.org/bytefmt v0.0.0-20180906201452-2aa6f33b730c
+	github.com/Microsoft/go-winio v0.4.12 // indirect
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
 	github.com/VertebrateResequencing/muxfys v3.0.5+incompatible
 	github.com/VividCortex/ewma v0.0.0-20170804035156-43880d236f69
 	github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f // indirect
+	github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/carbocation/runningvariance v0.0.0-20150817162428-fdcce8a03b6b
-	github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102
-	github.com/docker/distribution v0.0.0-20180327202408-83389a148052 // indirect
+	github.com/coreos/etcd v3.3.13+incompatible // indirect
+	github.com/coreos/go-etcd v2.0.0+incompatible // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/cpuguy83/go-md2man v1.0.10 // indirect
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2
+	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v0.0.0-20180524003928-df5175e1ee95
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.3.3 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
-	github.com/elazarl/goproxy v0.0.0-20181003060214-f58a169a71a5 // indirect
-	github.com/emicklei/go-restful v2.8.0+incompatible // indirect
+	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6 // indirect
-	github.com/fanatic/go-infoblox v0.0.0-20180819145836-7f3fc0bb3f6a
+	github.com/fanatic/go-infoblox v0.0.0-20190411220143-5d008d00551d
 	github.com/fatih/color v1.7.0
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-ini/ini v1.41.0 // indirect
-	github.com/go-ole/go-ole v0.0.0-20180625085808-7a0fa49edf48 // indirect
-	github.com/go-openapi/loads v0.18.0 // indirect
-	github.com/go-openapi/spec v0.18.0 // indirect
+	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 // indirect
+	github.com/go-ini/ini v1.42.0 // indirect
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/go-openapi/analysis v0.19.0 // indirect
+	github.com/go-openapi/errors v0.19.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.0 // indirect
+	github.com/go-openapi/jsonreference v0.19.0 // indirect
+	github.com/go-openapi/loads v0.19.0 // indirect
+	github.com/go-openapi/spec v0.19.0 // indirect
+	github.com/go-openapi/strfmt v0.19.0 // indirect
+	github.com/go-openapi/swag v0.19.0 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/gofrs/uuid v2.1.0+incompatible
-	github.com/gogo/protobuf v0.0.0-20180830160456-5669497fd644 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
-	github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 // indirect
-	github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect
-	github.com/google/go-cmp v0.2.0 // indirect
-	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible
+	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
+	github.com/golang/mock v1.3.1 // indirect
+	github.com/google/btree v1.0.0 // indirect
+	github.com/google/go-cmp v0.3.0 // indirect
+	github.com/google/gofuzz v1.0.0 // indirect
+	github.com/google/pprof v0.0.0-20190515194954-54271f7e092f // indirect
 	github.com/googleapis/gnostic v0.2.0 // indirect
-	github.com/gophercloud/gophercloud v0.0.0-20190116032514-258a61a0642d
-	github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect
+	github.com/gophercloud/gophercloud v0.0.0-20190520235722-e87e5f90e7e6
+	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
 	github.com/gorilla/websocket v1.4.0
-	github.com/gotestyourself/gotestyourself v2.1.0+incompatible // indirect
+	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
 	github.com/grafov/bcast v0.0.0-20161019100130-e9affb593f6c
-	github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f // indirect
-	github.com/hanwen/go-fuse v0.0.0-20190201110340-3342667fc815 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
+	github.com/hanwen/go-fuse v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/golang-lru v0.5.0
+	github.com/hashicorp/golang-lru v0.5.1
 	github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jinzhu/configor v0.0.0-20180614024415-4edaf76fe188
+	github.com/jinzhu/configor v1.0.0
 	github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7
-	github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3 // indirect
-	github.com/jtolds/gls v4.2.1+incompatible // indirect
+	github.com/json-iterator/go v1.1.6 // indirect
 	github.com/juju/ratelimit v1.0.1 // indirect
-	github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1
+	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
+	github.com/kisielk/errcheck v1.2.0 // indirect
 	github.com/kr/fs v0.1.0 // indirect
-	github.com/mattn/go-colorable v0.1.0 // indirect
-	github.com/mattn/go-isatty v0.0.4 // indirect
+	github.com/kr/pty v1.1.4 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 // indirect
+	github.com/mattn/go-colorable v0.1.1 // indirect
+	github.com/mattn/go-isatty v0.0.7 // indirect
 	github.com/minio/minio-go v6.0.14+incompatible // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
-	github.com/onsi/gomega v1.4.2 // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20190414153302-2ae31c8b6b30 // indirect
+	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/onsi/gomega v1.5.0 // indirect
 	github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 // indirect
 	github.com/opencontainers/image-spec v0.0.0-20180411145040-e562b0440392 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pborman/uuid v1.2.0 // indirect
+	github.com/pelletier/go-toml v1.4.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
-	github.com/pkg/errors v0.0.0-20180311214515-816c9085562c // indirect
-	github.com/pkg/sftp v0.0.0-20180824225003-b9345f483dc3
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/pkg/sftp v1.10.0
 	github.com/ricochet2200/go-disk-usage v0.0.0-20150921141558-f0d1b743428f
+	github.com/russross/blackfriday v2.0.0+incompatible // indirect
 	github.com/sasha-s/go-deadlock v0.2.0 // indirect
 	github.com/sb10/l15h v0.0.0-20170510122137-64c488bf8e22
-	github.com/sevlyar/go-daemon v0.0.0-20160925164401-01bb5caedcc4
-	github.com/shirou/gopsutil v0.0.0-20180901134234-eb1f1ab16f2e
+	github.com/sevlyar/go-daemon v0.1.1-0.20160925164401-01bb5caedcc4
+	github.com/shirou/gopsutil v2.18.12+incompatible
 	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
-	github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect
-	github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c
-	github.com/spf13/cobra v0.0.0-20180829160745-99dc12355885
+	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
+	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/cobra v0.0.3
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.3 // indirect
-	github.com/ugorji/go/codec v0.0.0-20180927125128-99ea80c8b19a
-	go.etcd.io/bbolt v1.3.1-etcd.7
-	golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc
-	golang.org/x/oauth2 v0.0.0-20190115181402-5dab4167f31c // indirect
-	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
+	github.com/spf13/viper v1.2.1 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/ugorji/go v1.1.4
+	github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 // indirect
+	go.etcd.io/bbolt v1.3.2
+	golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f
+	golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522 // indirect
+	golang.org/x/image v0.0.0-20190516052701-61b8692d9a5c // indirect
+	golang.org/x/lint v0.0.0-20190409202823-959b441ac422 // indirect
+	golang.org/x/mobile v0.0.0-20190509164839-32b2708ab171 // indirect
+	golang.org/x/net v0.0.0-20190520210107-018c4d40a106 // indirect
+	golang.org/x/oauth2 v0.0.0-20190517181255-950ef44c6e07 // indirect
+	golang.org/x/sys v0.0.0-20190520201301-c432e742b0af // indirect
+	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	golang.org/x/tools v0.0.0-20190520220859-26647e34d3c0 // indirect
+	google.golang.org/appengine v1.6.0 // indirect
+	google.golang.org/genproto v0.0.0-20190516172635-bb713bdc0e52 // indirect
+	google.golang.org/grpc v1.20.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.38.3 // indirect
+	gopkg.in/ini.v1 v1.42.0 // indirect
 	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect
-	gotest.tools v2.1.0+incompatible // indirect
-	k8s.io/api v0.0.0-20190202010521-49be0e3344fe
-	k8s.io/apimachinery v0.0.0-20190204010555-a98ff070d70e
-	k8s.io/client-go v10.0.0+incompatible
-	k8s.io/klog v0.1.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20190204220027-815fb30b1ecf // indirect
-	k8s.io/kubernetes v1.13.3 // indirect
+	gotest.tools v2.2.0+incompatible // indirect
+	honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a // indirect
+	k8s.io/api v0.0.0-20190515023547-db5a9d1c40eb
+	k8s.io/apimachinery v0.0.0-20190515023456-b74e4c97951f
+	k8s.io/client-go v11.0.0+incompatible
+	k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a // indirect
+	k8s.io/klog v0.3.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 // indirect
+	k8s.io/kubernetes v1.14.2 // indirect
 	nanomsg.org/go-mangos v0.0.0-20180815160134-b7ff4263f0d7
 	sigs.k8s.io/yaml v1.1.0 // indirect
 )
@@ -102,3 +144,7 @@ replace k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20180228050457-302974c
 replace k8s.io/api => k8s.io/api v0.0.0-20180308224125-73d903622b73
 
 replace k8s.io/client-go => k8s.io/client-go v7.0.0+incompatible
+
+replace github.com/grafov/bcast => github.com/grafov/bcast v0.0.0-20161019100130-e9affb593f6c
+
+replace github.com/sevlyar/go-daemon => github.com/sevlyar/go-daemon v0.1.1-0.20160925164401-01bb5caedcc4