Build & Module Update (#47)

* feat: remove upx from build to allow m1 development

* chore: update go dependencies and module version to 1.18

* feat: remove dependency on workerpool

Dockerfile

@@ -4,7 +4,7 @@
 FROM golang:alpine as base
 MAINTAINER  Stefan Martinov <[email protected]>
 
-RUN apk add --no-cache git make bash upx \
+RUN apk add --no-cache git \
   && rm -rf /var/cache/apk/*
 
 WORKDIR /app
@@ -24,8 +24,6 @@ MAINTAINER  Stefan Martinov <[email protected]>
 RUN GOARCH=amd64 GOOS=linux CGO_ENABLED=0 \
     go build -ldflags "-X main.Version=`git rev-parse --short HEAD`" -o /pagespeed_exporter pagespeed_exporter.go
 
-# strip and compress the binary
-RUN upx /pagespeed_exporter
 
 ##############################
 ###### STAGE: PACKAGE   ######

collector/scrape.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"net/http"
 	"runtime"
+	"sync"
 	"time"
 
-	"github.com/gammazero/workerpool"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/oauth2"
@@ -55,29 +55,39 @@ func (pss *pagespeedScrapeService) Scrape(parallel bool, requests []ScrapeReques
 		maxWorkers = runtime.NumCPU()
 	}
 
-	wp := workerpool.New(maxWorkers)
-
 	results := make(chan *ScrapeResult, 2*len(requests))
 
-	for _, req := range requests {
-		request := req
-		wp.Submit(func() {
-
-			scrape, err := pss.scrape(request)
-			if err != nil {
-				log.WithError(err).
-					WithFields(log.Fields{
-						"target":   request.Url,
-						"strategy": request.Strategy,
-					}).Warn("target scraping returned an error")
-				return
+	// Fill queue with scrape requests
+	requestChan := make(chan ScrapeRequest)
+	go func() {
+		for _, r := range requests {
+			requestChan <- r
+		}
+		close(requestChan)
+	}()
+
+	wg := sync.WaitGroup{}
+	wg.Add(maxWorkers)
+
+	for i := 0; i < maxWorkers; i++ {
+		go func() {
+			defer wg.Done()
+			for request := range requestChan {
+				scrape, err := pss.scrape(request)
+				if err != nil {
+					log.WithError(err).
+						WithFields(log.Fields{
+							"target":   request.Url,
+							"strategy": request.Strategy,
+						}).Warn("target scraping returned an error")
+					continue
+				}
+				results <- scrape
 			}
-			results <- scrape
-		})
-
+		}()
 	}
 
-	wp.StopWait()
+	wg.Wait()
 	close(results)
 
 	// Drain the channel after receiving all the results

collector/scrape_test.go

@@ -37,7 +37,7 @@ func Test_PagespeedScrapeService(t *testing.T) {
 		t.Fatalf("newPagespeedScrapeService should not throw an error: %v", err)
 	}
 
-	scrapes, err := service.Scrape(false, CalculateScrapeRequests("http://example.com/"))
+	scrapes, err := service.Scrape(true, CalculateScrapeRequests("http://example.com/"))
 	if err != nil {
 		t.Fatal("scrape should not throw an error")
 	}

go.mod

@@ -1,13 +1,36 @@
 module github.com/foomo/pagespeed_exporter
 
-go 1.12
+go 1.18
 
 require (
-	github.com/gammazero/workerpool v0.0.0-20190521015540-3b91a70bc0a1
-	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v1.5.1
-	github.com/sirupsen/logrus v1.4.2
+	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.12.2
+	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401
-	google.golang.org/api v0.82.0
+	golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
+	google.golang.org/api v0.83.0
+)
+
+require (
+	cloud.google.com/go/compute v1.6.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.34.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac // indirect
+	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )