Merge branch 'release/0.19.0'

.golangci.yml

@@ -0,0 +1,94 @@
+run:
+  deadline: 4m
+  build-tags:
+    - netgo
+  skip-files:
+    - jobqueue/static.go
+  modules-download-mode: readonly
+
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
+  format: colored-line-number
+
+linters-settings:
+  errcheck:
+    check-type-assertions: false
+    check-blank: true
+  govet:
+    check-shadowing: true
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    # local-prefixes: github.com/VertebrateResequencing/wr
+  maligned:
+    # print struct with more effective memory layout or not, false by default
+    suggest-new: true
+  unused:
+    # treat code as a program (not a library) and report unused exported identifiers; default is false.
+    # XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find funcs usages. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  unparam:
+    # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
+    # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  nakedret:
+    max-func-lines: 30
+  prealloc:
+    simple: true
+    range-loops: true
+    for-loops: true
+
+linters:
+  enable-all: true
+  # enable:
+  #   - deadcode
+  #   - errcheck
+  #   - gosimple
+  #   - govet
+  #   - ineffassign
+  #   - staticcheck
+  #   - structcheck
+  #   - typecheck
+  #   - unused
+  #   - varcheck
+  #   - goconst
+  #   - gocritic
+  #   - goimports
+  #   - golint
+  #   - gosec
+  #   - interfacer
+  #   - maligned
+  #   - misspell
+  #   - nakedret
+  #   - prealloc
+  #   - scopelint
+  #   - stylecheck
+  #   - unconvert
+  #   - unparam
+  disable-all: false
+  disable:
+    - depguard
+    - dupl
+    - gochecknoglobals
+    - gochecknoinits
+    - gocyclo
+    - gofmt
+    - lll
+
+issues:
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - gocyclo
+        - errcheck
+        - dupl
+        - gosec
+  max-issues-per-linter: 0
+  max-same-issues: 0
+  new: true
+

.golangci_extra.yml

@@ -0,0 +1,45 @@
+run:
+  deadline: 4m
+  build-tags:
+    - netgo
+  skip-files:
+    - jobqueue/static.go
+  modules-download-mode: readonly
+
+output:
+  format: colored-line-number
+
+linters-settings:
+  gocyclo:
+    # minimal code complexity to report, 30 by default (but we recommend 10-20)
+    min-complexity: 10
+  dupl:
+    # tokens count to trigger issue, 150 by default
+    threshold: 100
+
+linters:
+  disable-all: true
+  enable-all: false
+  enable:
+    - dupl
+    - gocyclo
+    - gochecknoglobals
+    - gochecknoinits
+
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - gocyclo
+        - dupl
+        - gochecknoglobals
+        - gochecknoinits
+    - path: cmd/
+      linters:
+        - gocyclo
+        - dupl
+        - gochecknoglobals
+        - gochecknoinits
+
+  new: true
+

.travis.yml

@@ -13,7 +13,7 @@ env:
     - WR_MANAGERPORT="11301"
     - GO111MODULE=on
 go:
-  - "1.12.5"
+  - "1.12.7"
 go_import_path: github.com/VertebrateResequencing/wr
 install:
   - "go mod verify"

.vscode/settings.json

@@ -1,35 +1,17 @@
 {
-  "go.lintOnSave": "package",
-  "editor.formatOnSave": true,
-  "go.lintTool": "gometalinter",
-  "go.vetOnSave": "workspace",
-  "go.formatTool": "goimports",
-  "go.lintFlags": [
-    "--vendor",
-    "--disable-all",
-    "--enable=unconvert",
-    "--enable=interfacer",
-    "--enable=gotypex",
-    "--enable=structcheck",
-    "--enable=gofmt",
-    "--enable=varcheck",
-    "--enable=gotype",
-    "--enable=goconst",
-    "--enable=deadcode",
-    "--enable=vet",
-    "--enable=unparam",
-    "--enable=golint",
-    "--enable=goimports",
-    "--enable=gosec",
-    "--enable=megacheck",
-    "--enable=ineffassign",
-    "--enable=misspell",
-    "--enable=nakedret",
-    "--enable=vetshadow",
-    "--enable=errcheck",
-    "--json",
-    "--fast",
-    "--deadline",
-    "30s"
-  ]
+    "[go]": {
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": true
+        },
+    },
+    "go.formatTool": "goimports",
+    "go.formatFlags": [
+        "-w"
+    ],
+    "go.lintOnSave": "package",
+    "go.lintTool": "golangci-lint",
+    "go.lintFlags": [
+        "--fast"
+    ]
 }

CHANGELOG.md

@@ -5,6 +5,46 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this
 project adheres to [Semantic Versioning](http://semver.org/).
 
 
+## [0.19.0] - 2019-08-01
+### Added
+- New --cloud_auto_confirm_dead option to `wr manager start` (and similar
+  option for `wr cloud deploy`, and new config option) that allows dead
+  OpenStack servers to be automatically confirmed dead (destroying them, and
+  confirming the jobs running on them as dead as well) after, by default,
+  30mins. NB: for the old behaviour where dead servers were left alone until
+  you manually investigated, set this option to 0.
+
+### Changed
+- OpenStack scheduler reimplemented for new fixes and behaviour:
+  1. Avoids a lock-up issue where jobs would stop getting scheduled under
+     certain circumstances.
+  2. Quota warnings no longer appear until quota is physically used up.
+  3. The flavor of server to spawn and the jobs to run on them are reassessed
+     after every spawn, job completion and schedule, allowing bin-packing to do
+     the expected thing as new jobs are scheduled over time.
+  4. Spawned servers that are no longer needed are abandoned right away,
+     allowing any other still-desired server to come up straight away.
+  5. Servers for different kinds of job (according to having different resource
+     requirements) are now spawned fully simultaneously, while servers for the
+     same kind of job are now spawned fully sequentially. (The old behaviour was
+     servers for all kinds of jobs were spawned sequentially, but with some
+     overlap.)
+- Cloud scripts (being the scripts that run after an OpenStack server boots up)
+  now have a time limit of 15 mins, so that scripts that fail to exit do not
+  cause servers to be created that are never used and never destroyed.
+- For developers of wr, the linting method has changed to golangci-lint. See
+  comment in Makefile for installation instructions.
+
+### Fixed
+- Theoretical edge-case bugs fixed, alongside potential general stability
+  improvements.
+- Memory leak associated with running jobs that write empty files to uncached S3
+  mounts.
+- Runners now wait the appropriate time for new jobs to run, and start running
+  them the moment they're added.
+- Edge-case where an OpenStack server could scale down while running a command.
+
+
 ## [0.18.1] - 2019-05-23
 ### Added
 - New `--manager_flavor` option to `wr cloud deploy` (defaulting to new

Makefile

@@ -43,20 +43,12 @@ race:
 	@go test -p 1 -tags netgo -race -v --count 1 ./rp
 	@go test -p 1 -tags netgo -race -v --count 1 ./limiter
 
-# cd $HOME/go && curl -L https://git.io/vp6lP | sh
-# until all go tools have module support:
-# mkdir -p $HOME/go/src
-# go mod vendor
-# rsync -a vendor/ ~/go/src/
-# rm -fr vendor
-# ln -s $PWD $HOME/go/src/github.com/VertebrateResequencing/wr
-lint: export GO111MODULE = off
+# curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.16.0
 lint:
-	@gometalinter --vendor --aggregate --deadline=240s ./... | sort
+	@golangci-lint run
 
-lintextra: export GO111MODULE = off
 lintextra:
-	@gometalinter --vendor --aggregate --deadline=240s --disable-all --enable=gocyclo --enable=dupl ./... | sort
+	@golangci-lint run -c .golangci_extra.yml
 
 clean:
 	@rm -f ./wr

README.md

@@ -51,8 +51,9 @@ Alternatively, build it yourself (at least v1.12 of go is required):
 An example way of setting up a personal Go installation in your home directory
 would be:
 
-        wget https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz
-        tar -xvzf go1.12.5.linux-amd64.tar.gz && rm go1.12.5.linux-amd64.tar.gz
+        export GOV=1.12.7
+        wget https://dl.google.com/go/go$GOV.linux-amd64.tar.gz
+        tar -xvzf go$GOV.linux-amd64.tar.gz && rm go$GOV.linux-amd64.tar.gz
         export PATH=$PATH:$HOME/go/bin
 
 2. Download, compile, and install wr (not inside $GOPATH, if you set that):