Merge branch 'master' into patch-1

Dockerfile

@@ -1,13 +1,18 @@
-FROM golang:alpine
+FROM golang:alpine as build-env
 
-RUN apk add --no-cache --virtual git && \
-    go-wrapper download github.com/messagebird/beanstalkd_exporter && \
-    cp -v $GOPATH/bin/beanstalkd_exporter /usr/local/bin/beanstalkd_exporter && \
-    rm -rvf $GOPATH && \
-    apk del git
+RUN apk add git
 
-COPY examples/ /etc/beanstalkd_exporter/
+# Copy source + vendor
+COPY . /go/src/github.com/messagebird/beanstalkd_exporter
+WORKDIR /go/src/github.com/messagebird/beanstalkd_exporter
+
+# Build
+ENV GOPATH=/go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=off go build -v -a -ldflags "-s -w" -o /go/bin/beanstalkd_exporter .
 
-EXPOSE 8080
+# Build mininal image with compiled app
+FROM scratch
+COPY examples/ /etc/beanstalkd_exporter/
+COPY --from=build-env /go/bin/beanstalkd_exporter /usr/bin/beanstalkd_exporter
 ENTRYPOINT ["beanstalkd_exporter"]
-CMD ["-beanstalkd.address", "beanstalkd:11300", "-mapping-config", "/etc/beanstalkd_exporter/mapping.conf"]
+CMD ["-beanstalkd.address", "beanstalkd:11300"]

README.md

@@ -7,17 +7,27 @@ Beanstalkd Exporter is a [beanstalkd](http://kr.github.io/beanstalkd/) stats exp
 
 ## How does it work?
 
-Every now and then a goroutine connects to a beanstalkd server and asks for stats. These stats are then stored in memory for later.
+Every now and then, Prometheus will request a "scrape" of metrics from
+this application via an HTTP request to /metrics. During this scrape
+request the exporter will connect to beanstalk and ask for stats. Stats
+are fetched for the whole instance and for each individual tube.
 
-Every now and then Prometheus requests for stats. We read those from memory and respond.
+If you have many tubes and fetching stats one-by-one takes longer than
+your allowed scrape duration configured in prometheus, you can increase
+the number of concurrent tube stats workers via the
+`-num-tube-stat-workers` flag, to parallelize the work required.
 
 ## Usage
 
-Running beanstalkd_exporter is as easy as executing `beanstalkd_exporter` on the command line.  One argument is required: `-mapping-config` (see below for what it needs).
+
+Running beanstalkd_exporter is as easy as executing `beanstalkd_exporter` on the command line. One argument is required: `-mapping-config` (see below for what it needs).
+
 ```bash
 $ beanstalkd_exporter -config examples/servers.conf -mapping-config examples/mapping.conf
 ```
+
 Use the -h flag to get help information.
+
 ```bash
 $ beanstalkd_exporter -h
 Usage of ./bin/beanstalkd_exporter:
@@ -31,6 +41,8 @@ Usage of ./bin/beanstalkd_exporter:
     	The number of seconds that we poll the beanstalkd server for stats. (default 30)
   -sleep-between-tube-stats int
     	The number of milliseconds to sleep between tube stats. (default 5000)
+  -num-tube-stat-workers int
+    	The number of concurrent workers to use to fetch tube stats. (default 1)
   -web.listen-address string
     	Address to listen on for web interface and telemetry. (default ":8080")
   -web.telemetry-path string

exporter.go

@@ -159,6 +159,7 @@ func (e *Exporter) scrape(f func(prometheus.Collector)) {
 	if *logLevel == "debug" {
 		log.Debugf("Debug: Calling %s ListTubes()", e.address)
 	}
+
 	// stat every tube
 	tubes, err := c.ListTubes()
 	if err != nil {
@@ -168,11 +169,43 @@ func (e *Exporter) scrape(f func(prometheus.Collector)) {
 	}
 	e.scrapeCountMetric.WithLabelValues("success").Inc()
 
-	// for every tube
+	// spin out workers to fetch metrics for every tube
+	wg := &sync.WaitGroup{}
+	chtubes := make(chan string)
+	for i := 0; i < *numTubeStatWorkers; i++ {
+		go e.scrapeWorker(i, c, wg, f, chtubes)
+		wg.Add(1)
+	}
+
+	// queue up tubes to be fetched
 	for _, name := range tubes {
+		chtubes <- name
+	}
+
+	// wait for everything to finish
+	close(chtubes)
+	wg.Wait()
+}
+
+func (e *Exporter) scrapeWorker(i int, c *beanstalk.Conn, wg *sync.WaitGroup, f func(prometheus.Collector), ch <-chan string) {
+	defer wg.Done()
+
+	if *logLevel == "debug" {
+		log.Debugf("Debug: scrape worker %d started", i)
+	}
+
+	for name := range ch {
+		if *logLevel == "debug" {
+			log.Debugf("Debug: scrape worker %d fetching tube %s", i, name)
+		}
+
 		e.statTube(c, name, f)
 		time.Sleep(time.Duration(*sleepBetweenStats) * time.Millisecond)
 	}
+
+	if *logLevel == "debug" {
+		log.Debugf("Debug: scrape worker %d finished", i)
+	}
 }
 
 func (e *Exporter) statTube(c *beanstalk.Conn, tubeName string, f func(prometheus.Collector)) {
@@ -186,7 +219,6 @@ func (e *Exporter) statTube(c *beanstalk.Conn, tubeName string, f func(prometheu
 		labels = mappedLabels
 		labels["tube"] = labels["name"]
 		delete(labels, "name")
-
 	} else {
 		labels = prometheus.Labels{"tube": tubeName}
 	}

go.mod

@@ -0,0 +1,21 @@
+module github.com/messagebird/beanstalkd_exporter
+
+go 1.12
+
+require (
+	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
+	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
+	github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a // indirect
+	github.com/golang/protobuf v0.0.0-20170920220647-130e6b02ab05 // indirect
+	github.com/howeyc/fsnotify v0.9.0
+	github.com/kr/beanstalk v0.0.0-20150923205605-e99e1a384e4a
+	github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect
+	github.com/prometheus/client_golang v0.8.0
+	github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612 // indirect
+	github.com/prometheus/common v0.0.0-20171006141418-1bab55dd05db
+	github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
+	golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 // indirect
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
+)

go.sum

@@ -0,0 +1,36 @@
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c=
+github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v0.0.0-20170920220647-130e6b02ab05 h1:Kesru7U6Mhpf/x7rthxAKnr586VFmoE2NdEvkOKvfjg=
+github.com/golang/protobuf v0.0.0-20170920220647-130e6b02ab05/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
+github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/beanstalk v0.0.0-20150923205605-e99e1a384e4a h1:obXkCagFdhty4KYKpeQzDStWQih/ITPDa34jHevsQVE=
+github.com/kr/beanstalk v0.0.0-20150923205605-e99e1a384e4a/go.mod h1:S640fId9Ag4k2hh6Hwwj62pMSZqfMtg/kfKPeAOhET8=
+github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc=
+github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612 h1:13pIdM2tpaDi4OVe24fgoIS7ZTqMt0QI+bwQsX5hq+g=
+github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20171006141418-1bab55dd05db h1:PmL7nSW2mvuotGlJKuvUcSI/eE86zwYUcIAGoB6eHBk=
+github.com/prometheus/common v0.0.0-20171006141418-1bab55dd05db/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13 h1:leRfx9kcgnSDkqAFhaaUcRqpAZgnFdwZkZcdRcea1h0=
+github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U=
+golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

main.go

@@ -11,12 +11,13 @@ import (
 )
 
 var (
-	address           = flag.String("beanstalkd.address", "localhost:11300", "Beanstalkd server address")
-	logLevel          = flag.String("log.level", "warning", "The log level.")
-	mappingConfig     = flag.String("mapping-config", "", "A file that describes a mapping of tube names.")
-	sleepBetweenStats = flag.Int("sleep-between-tube-stats", 5000, "The number of milliseconds to sleep between tube stats.")
-	listenAddress     = flag.String("web.listen-address", ":8080", "Address to listen on for web interface and telemetry.")
-	metricsPath       = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
+	address            = flag.String("beanstalkd.address", "localhost:11300", "Beanstalkd server address")
+	logLevel           = flag.String("log.level", "warning", "The log level.")
+	mappingConfig      = flag.String("mapping-config", "", "A file that describes a mapping of tube names.")
+	sleepBetweenStats  = flag.Int("sleep-between-tube-stats", 5000, "The number of milliseconds to sleep between tube stats.")
+	numTubeStatWorkers = flag.Int("num-tube-stat-workers", 1, "The number of concurrent workers to use to fetch tube stats.")
+	listenAddress      = flag.String("web.listen-address", ":8080", "Address to listen on for web interface and telemetry.")
+	metricsPath        = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
 )
 
 var (
@@ -60,6 +61,10 @@ func watchConfig(fileName string, mapper *tubeMapper) {
 func main() {
 	flag.Parse()
 
+	if *logLevel == "debug" {
+		log.Base().SetLevel("debug")
+	}
+
 	mapper = newTubeMapper()
 	if *mappingConfig != "" {
 		err := mapper.initFromFile(*mappingConfig)
@@ -78,14 +83,18 @@ func main() {
 	))
 
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write([]byte(`<html>
-             <head><title>Beanstalkd Exporter</title></head>
-             <body>
-             <h1>Beanstalkd Exporter</h1>
-             <p><a href='` + *metricsPath + `'>Metrics</a></p>
-             </body>
-             </html>`))
+		w.Write([]byte(`
+			<html>
+              <head><title>Beanstalkd Exporter</title></head>
+              <body>
+                <h1>Beanstalkd Exporter</h1>
+                <p><a href='` + *metricsPath + `'>Metrics</a></p>
+              </body>
+            </html>
+		`),
+		)
 	})
+
 	log.Warnf("Listening on port %s .", *listenAddress)
 	log.Fatal(http.ListenAndServe(*listenAddress, nil))
 }