Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More isolation per aws-checker instance for testing #11

Merged
merged 1 commit into from
Oct 11, 2024
Merged

More isolation per aws-checker instance for testing #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 43 additions & 26 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,6 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
requestDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "aws_request_duration_seconds",
Help: "Time spent in requests for aws.",
Buckets: prometheus.ExponentialBuckets(0.01, 2, 10),
},
[]string{"service", "method", "status"},
)
)

func init() {
prometheus.MustRegister(requestDuration)
}

func main() {
// Create a channel to receive OS signals
sigs := make(chan os.Signal, 1)
Expand Down Expand Up @@ -62,13 +47,41 @@ func ContextWithSignal(ctx context.Context, sigs chan os.Signal) context.Context

func Run(ctx context.Context, opts ...Option) error {
var (
requestDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "aws_request_duration_seconds",
Help: "Time spent in requests for aws.",
Buckets: prometheus.ExponentialBuckets(0.01, 2, 10),
},
[]string{"service", "method", "status"},
)

registry = prometheus.NewRegistry()

httpServerGracefulShutdownTimeout = 5 * time.Second

httpServer = &http.Server{Addr: ":8080"}
listenErr = make(chan error, 1)
httpMux = http.NewServeMux()
httpServer = &http.Server{
Addr: ":8080",
Handler: httpMux,
}
listenErr = make(chan error, 1)
)

http.Handle("/metrics", promhttp.Handler())
registry.MustRegister(requestDuration)
defer func() {
if ok := registry.Unregister(requestDuration); !ok {
log.Printf("failed to unregister requestDuration: it was not registered")
}
}()

// This is the same as getting the default handler using promhttp.Handler()
// but with our own registry instead of the promhttp's default registry.
promHttpHandler := promhttp.InstrumentMetricHandler(
registry, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
)

httpMux.Handle("/metrics", promHttpHandler)
go func() {
listenErr <- httpServer.ListenAndServe()
}()
Expand All @@ -78,7 +91,7 @@ func Run(ctx context.Context, opts ...Option) error {
return fmt.Errorf("unable to load SDK config, %v", err)
}

chkr := newChecker(cfg, opts...)
chkr := newChecker(cfg, requestDuration, opts...)
checkCtx, checkCancel := context.WithCancel(ctx)

for {
Expand Down Expand Up @@ -109,6 +122,8 @@ func Run(ctx context.Context, opts ...Option) error {
}

type checker struct {
requestDuration *prometheus.HistogramVec

s3Client *s3.Client
dynamoClient *dynamodb.Client
sqsClient *sqs.Client
Expand All @@ -125,12 +140,14 @@ type checker struct {

type Option func(*checker)

func newChecker(cfg aws.Config, opts ...Option) *checker {
func newChecker(cfg aws.Config, requestDuration *prometheus.HistogramVec, opts ...Option) *checker {
c := &checker{}
for _, opt := range opts {
opt(c)
}

c.requestDuration = requestDuration

c.s3Client = s3.NewFromConfig(cfg, c.s3Opts...)
c.dynamoClient = dynamodb.NewFromConfig(cfg, c.dynamodbOpts...)
c.sqsClient = sqs.NewFromConfig(cfg, c.sqsOpts...)
Expand All @@ -156,9 +173,9 @@ func (c *checker) doCheck(ctx context.Context) {
return
} else if err != nil {
log.Printf("failed to get object, %v", err)
requestDuration.WithLabelValues("S3", "GetObject", "Failure").Observe(getDuration)
c.requestDuration.WithLabelValues("S3", "GetObject", "Failure").Observe(getDuration)
} else {
requestDuration.WithLabelValues("S3", "GetObject", "Success").Observe(getDuration)
c.requestDuration.WithLabelValues("S3", "GetObject", "Success").Observe(getDuration)
}

// DynamoDB Scan
Expand All @@ -172,9 +189,9 @@ func (c *checker) doCheck(ctx context.Context) {
return
} else if err != nil {
log.Printf("failed to get item, %v", err)
requestDuration.WithLabelValues("DynamoDB", "Scan", "Failure").Observe(dynamoDuration)
c.requestDuration.WithLabelValues("DynamoDB", "Scan", "Failure").Observe(dynamoDuration)
} else {
requestDuration.WithLabelValues("DynamoDB", "Scan", "Success").Observe(dynamoDuration)
c.requestDuration.WithLabelValues("DynamoDB", "Scan", "Success").Observe(dynamoDuration)
}

// SQS ReceiveMessage
Expand All @@ -188,8 +205,8 @@ func (c *checker) doCheck(ctx context.Context) {
return
} else if err != nil {
log.Printf("failed to receive message, %v", err)
requestDuration.WithLabelValues("SQS", "ReceiveMessage", "Failure").Observe(sqsDuration)
c.requestDuration.WithLabelValues("SQS", "ReceiveMessage", "Failure").Observe(sqsDuration)
} else {
requestDuration.WithLabelValues("SQS", "ReceiveMessage", "Success").Observe(sqsDuration)
c.requestDuration.WithLabelValues("SQS", "ReceiveMessage", "Success").Observe(sqsDuration)
}
}
32 changes: 15 additions & 17 deletions main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,23 +47,21 @@ func TestRun(t *testing.T) {
})
})

// The below fails like the below, due to the duplicate MustRegister call to the prometheus library:
// panic: pattern "/metrics" (registered at /go/aws-checker/main.go:71) conflicts with pattern "/metrics" (registered at /go/aws-checker/main.go:71):
// t.Run("s3 failing", func(t *testing.T) {
// checkRun(t, testcase{
// setupS3: false,
// okLabels: []labels{
// {"S3", "GetObject", "Failure"},
// {"SQS", "ReceiveMessage", "Success"},
// {"DynamoDB", "Scan", "Success"},
// },
// ngLabels: []labels{
// {"S3", "GetObject", "Success"},
// {"SQS", "ReceiveMessage", "Failure"},
// {"DynamoDB", "Scan", "Failure"},
// },
// })
// })
t.Run("s3 failing", func(t *testing.T) {
checkRun(t, testcase{
setupS3: false,
okLabels: []labels{
{"S3", "GetObject", "Failure"},
{"SQS", "ReceiveMessage", "Success"},
{"DynamoDB", "Scan", "Success"},
},
ngLabels: []labels{
{"S3", "GetObject", "Success"},
{"SQS", "ReceiveMessage", "Failure"},
{"DynamoDB", "Scan", "Failure"},
},
})
})
}

func checkRun(t *testing.T, tc testcase) {
Expand Down