diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ef498c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/vendor +/cost-exporter* diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2d0e26e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +sudo: required + +language: go + +services: + - docker + +script: +- docker build -t cost-exporter --target builder --no-cache . +- > + docker run + --name cost-exporter + --entrypoint "sh" + -e CGO_ENABLED=0 + --workdir "/go/src/github.com/rebuy-de/cost-exporter" + cost-exporter + -euxc "make xc && mkdir releases && mv cost-exporter-* releases" +- docker cp -L cost-exporter:/go/src/github.com/rebuy-de/cost-exporter/releases ./releases +- ls -l * + +deploy: + provider: releases + api_key: $GITHUB_TOKEN + file_glob: true + file: releases/* + skip_cleanup: true + on: + repo: rebuy-de/cost-exporter + tags: true diff --git a/Dockerfile b/Dockerfile index 7ffe011..15cb9b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1,27 @@ -FROM golang:1.10-alpine +FROM golang:1.10-alpine as builder + +RUN apk add --no-cache git make + +# Configure Go +ENV GOPATH /go +ENV PATH /go/bin:$PATH +RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin + +# Install Go Tools +RUN go get -u github.com/golang/lint/golint +RUN go get -u github.com/golang/dep/cmd/dep + +COPY . /go/src/github.com/rebuy-de/cost-exporter +WORKDIR /go/src/github.com/rebuy-de/cost-exporter +RUN CGO_ENABLED=0 make install + +FROM alpine:latest + +RUN apk add --no-cache ca-certificates + +COPY --from=builder /go/bin/cost-exporter /usr/local/bin/ +COPY run.sh /run.sh + +RUN chmod +x /run.sh + +ENTRYPOINT ["/run.sh"] diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..f4eac24 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,238 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + digest = "1:bee95eb17352bd55583a5f4646cac5395f1f1e7de5c907d8889efd13896cea98" + name = "github.com/aws/aws-sdk-go" + packages = [ + "aws", + "aws/awserr", + "aws/awsutil", + "aws/client", + "aws/client/metadata", + "aws/corehandlers", + "aws/credentials", + "aws/credentials/ec2rolecreds", + "aws/credentials/endpointcreds", + "aws/credentials/stscreds", + "aws/csm", + "aws/defaults", + "aws/ec2metadata", + "aws/endpoints", + "aws/request", + "aws/session", + "aws/signer/v4", + "internal/sdkio", + "internal/sdkrand", + "internal/sdkuri", + "internal/shareddefaults", + "private/protocol", + "private/protocol/ec2query", + "private/protocol/json/jsonutil", + "private/protocol/jsonrpc", + "private/protocol/query", + "private/protocol/query/queryutil", + "private/protocol/rest", + "private/protocol/xml/xmlutil", + "service/costexplorer", + "service/ec2", + "service/sts", + ] + pruneopts = "" + revision = "b36008bfc7d4b9826423278ae193420e41afc4b4" + version = "v1.15.14" + +[[projects]] + branch = "master" + digest = "1:c0bec5f9b98d0bc872ff5e834fac186b807b656683bd29cb82fb207a1513fabb" + name = "github.com/beorn7/perks" + packages = ["quantile"] + pruneopts = "" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + digest = "1:858b7fe7b0f4bc7ef9953926828f2816ea52d01a88d72d1c45bc8c108f23c356" + name = "github.com/go-ini/ini" + packages = ["."] + pruneopts = "" + revision = "358ee7663966325963d4e8b2e1fbd570c5195153" + version = "v1.38.1" + +[[projects]] + digest = "1:f958a1c137db276e52f0b50efee41a1a389dcdded59a69711f3e872757dab34b" + name = "github.com/golang/protobuf" + packages = ["proto"] + pruneopts = "" + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" + +[[projects]] + digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + pruneopts = "" + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e" + name = "github.com/jmespath/go-jmespath" + packages = ["."] + pruneopts = "" + revision = "0b12d6b5" + +[[projects]] + digest = "1:63722a4b1e1717be7b98fc686e0b30d5e7f734b9e93d7dee86293b6deab7ea28" + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + pruneopts = "" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" + name = "github.com/pkg/errors" + packages = ["."] + pruneopts = "" + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + digest = "1:4142d94383572e74b42352273652c62afec5b23f325222ed09198f46009022d1" + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/promhttp", + ] + pruneopts = "" + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" + +[[projects]] + branch = "master" + digest = "1:185cf55b1f44a1bf243558901c3f06efa5c64ba62cfdcbb1bf7bbe8c3fb68561" + name = "github.com/prometheus/client_model" + packages = ["go"] + pruneopts = "" + revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" + +[[projects]] + branch = "master" + digest = "1:f477ef7b65d94fb17574fc6548cef0c99a69c1634ea3b6da248b63a61ebe0498" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + ] + pruneopts = "" + revision = "c7de2306084e37d54b8be01f3541a8464345e9a5" + +[[projects]] + branch = "master" + digest = "1:e04aaa0e8f8da0ed3d6c0700bd77eda52a47f38510063209d72d62f0ef807d5e" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs", + ] + pruneopts = "" + revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" + +[[projects]] + digest = "1:13f72d28227aaed50d9a1226efbd52d478fc4cac213ed47b175ab469e11faf9b" + name = "github.com/rebuy-de/rebuy-go-sdk" + packages = ["cmdutil"] + pruneopts = "" + revision = "ca80dbe1ba46d4f835698698bba63b1b48385a41" + version = "v1.2.0" + +[[projects]] + digest = "1:6ab228f39a195cb1dab3564a0f27dc24a52bb3a19fa58dd2967f1e7b2482d82b" + name = "github.com/robfig/cron" + packages = ["."] + pruneopts = "" + revision = "b41be1df696709bb6395fe435af20370037c0b4c" + version = "v1.1" + +[[projects]] + digest = "1:3fcbf733a8d810a21265a7f2fe08a3353db2407da052b233f8b204b5afc03d9b" + name = "github.com/sirupsen/logrus" + packages = ["."] + pruneopts = "" + revision = "3e01752db0189b9157070a0e1668a620f9a85da2" + version = "v1.0.6" + +[[projects]] + digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6" + name = "github.com/spf13/cobra" + packages = ["."] + pruneopts = "" + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" + +[[projects]] + digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e" + name = "github.com/spf13/pflag" + packages = ["."] + pruneopts = "" + revision = "9a97c102cda95a86cec2345a6f09f55a939babf5" + version = "v1.0.2" + +[[projects]] + branch = "master" + digest = "1:97fb4f02b231deef44704b6fa74d31e46f669fb217f192754ba603c2b32a190b" + name = "golang.org/x/crypto" + packages = ["ssh/terminal"] + pruneopts = "" + revision = "aabede6cba87e37f413b3e60ebfc214f8eeca1b0" + +[[projects]] + branch = "master" + digest = "1:1132cbdac95d6a2a30f7434f7190b3b1345e85b6071e74911b1dce3e19e1543d" + name = "golang.org/x/sys" + packages = [ + "unix", + "windows", + ] + pruneopts = "" + revision = "1c9583448a9c3aa0f9a6a5241bf73c0bd8aafded" + +[[projects]] + digest = "1:50dc9975542e0e7e7444e4683011041b1386a42db360a2f4e2ed8edb34e7e4c9" + name = "gopkg.in/gemnasium/logrus-graylog-hook.v2" + packages = ["."] + pruneopts = "" + revision = "c4a7de647eaaac00ffea599b7773e5e6804fa3b7" + version = "v2.0.7" + +[[projects]] + digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2" + name = "gopkg.in/yaml.v2" + packages = ["."] + pruneopts = "" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/aws/aws-sdk-go/aws", + "github.com/aws/aws-sdk-go/aws/credentials", + "github.com/aws/aws-sdk-go/aws/endpoints", + "github.com/aws/aws-sdk-go/aws/session", + "github.com/aws/aws-sdk-go/service/costexplorer", + "github.com/aws/aws-sdk-go/service/ec2", + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/rebuy-de/rebuy-go-sdk/cmdutil", + "github.com/robfig/cron", + "github.com/sirupsen/logrus", + "github.com/spf13/cobra", + "gopkg.in/yaml.v2", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..22d6652 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,15 @@ +[[constraint]] + name = "github.com/rebuy-de/rebuy-go-sdk" + version = "1.2.0" + +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "0.8.0" + +[[constraint]] + name = "github.com/aws/aws-sdk-go" + version = "1.15.4" + +[[constraint]] + name = "github.com/robfig/cron" + version = "1.1.0" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1061a37 --- /dev/null +++ b/Makefile @@ -0,0 +1,3 @@ +PACKAGE=github.com/rebuy-de/cost-exporter + +include golang.mk diff --git a/README.md b/README.md index bc3dd28..e833106 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,46 @@ # cost-exporter -Retrieves cost metrics and core counts from the AWS API and exposes this information via a Prometheus /metrics endpoint. + +[![Build Status](https://travis-ci.org/rebuy-de/cost-exporter.svg?branch=master)](https://travis-ci.org/rebuy-de/cost-exporter) +[![license](https://img.shields.io/github/license/rebuy-de/cost-exporter.svg)]() + +Retrieves cost metrics and core counts from the AWS API and exposes this information via a Prometheus `/metrics` endpoint. + +> **Development Status** *cost-exporter* is for internal use only. Feel free to use +> it, but expect big unanounced changes at any time. Furthermore we are very +> restrictive about code changes, therefore you should communicate any changes +> before working on an issue. + + +## Installation + +Docker containers are are provided [here](https://quay.io/repository/rebuy/cost-exporter). To obtain the latest docker image run `docker pull quay.io/rebuy/cost-exporter:master`. + +To compile *cost-exporter* from source you need a working +[Golang](https://golang.org/doc/install) development environment. The sources +must be cloned to `$GOPATH/src/github.com/rebuy-de/cost-exporter`. + +Also you need to install [godep](github.com/golang/dep/cmd/dep), +[golint](https://github.com/golang/lint/) and [GNU +Make](https://www.gnu.org/software/make/). + +Then you just need to run `make build` to compile a binary into the project +directory or `make install` to install *cost-exporter* into `$GOPATH/bin`. With +`make xc` you can cross compile *cost-exporter* for other platforms. + + +## Usage + +**node-drainer**'s configuration is done using an configuration file that is pointed to when running the command, as well as a flag that defines the port it should listen on: +``` +cost-exporter --config=/cost-exporter/config.yaml --port=8080 +``` + +For more information, run: +``` +cost-exporter --help +``` + + +### Running in Kubernetes + +Please see the `example/k8s/` directory for an example of how to run `cost-exporter` in Kubernetes. diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..6bd1a9b --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "github.com/rebuy-de/cost-exporter/pkg/config" + "github.com/rebuy-de/cost-exporter/pkg/prom" + "github.com/rebuy-de/cost-exporter/pkg/retriever" + "github.com/rebuy-de/rebuy-go-sdk/cmdutil" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +type App struct { + config string + port string +} + +func (app *App) Run(cmd *cobra.Command, args []string) { + if app.config == "" { + logrus.Fatal("Configuration file location not defined.") + } + + config := config.Parse(app.config) + + coreRetriever := retriever.CoreRetriever{ + Accounts: config.Accounts, + IntervalSec: config.Settings.CoresInterval, + } + coreRetriever.Run() + + costRetriever := retriever.CostRetriever{ + Accounts: config.Accounts, + Cron: config.Settings.CostCron, + } + costRetriever.Run() + + prom.Run(app.port) + + select {} +} + +func (app *App) Bind(cmd *cobra.Command) { + cmd.PersistentFlags().StringVarP( + &app.config, "config", "c", "", `Path to configuration file.`) + cmd.PersistentFlags().StringVarP( + &app.port, "port", "p", "8080", `Port to bind to.`) +} + +func NewRootCommand() *cobra.Command { + cmd := cmdutil.NewRootCommand(new(App)) + cmd.Short = "AWS billing data Prometheus exporter." + cmd.Use = "cost-exporter" + return cmd +} diff --git a/example/k8s/configmap.yaml b/example/k8s/configmap.yaml new file mode 100644 index 0000000..cb758f4 --- /dev/null +++ b/example/k8s/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cost-exporter + labels: + app: cost-exporter +data: + config.yaml: |- + accounts: + - name: staging + id: ABCDEFGHIJKLMNOPQRST + secret: 1234567890123456789012345678901234567890 + settings: + costCron: "0 0 19 * * *" + coresInterval: 300 diff --git a/example/k8s/deployment.yaml b/example/k8s/deployment.yaml new file mode 100644 index 0000000..e1aa992 --- /dev/null +++ b/example/k8s/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: extensions/v1beta1 +kind: Deployment + +metadata: + name: cost-exporter + labels: + app: cost-exporter + +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 0 + + selector: + matchLabels: + app: cost-exporter + + template: + metadata: + name: cost-exporter + labels: + app: cost-exporter + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + + spec: + containers: + - name: silo + image: quay.io/rebuy/cost-exporter:master + imagePullPolicy: Always + args: + - '--config=/cost-exporter/config.yaml' + - '--port=8080' + ports: + - containerPort: 8080 + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 10m + memory: 50Mi + volumeMounts: + - name: config-volume + mountPath: /cost-exporter + volumes: + - name: config-volume + configMap: + name: cost-exporter diff --git a/golang.mk b/golang.mk new file mode 100644 index 0000000..de4f571 --- /dev/null +++ b/golang.mk @@ -0,0 +1,84 @@ +# Source: https://github.com/rebuy-de/golang-template +# Version: 2.0.4-snapshot +# Dependencies: +# * dep (https://github.com/golang/dep) +# * gocov (https://github.com/axw/gocov) +# * gocov-html (https://github.com/matm/gocov-html) + +NAME=$(notdir $(PACKAGE)) + +BUILD_VERSION=$(shell git describe --always --dirty --tags | tr '-' '.' ) +BUILD_DATE=$(shell date) +BUILD_HASH=$(shell git rev-parse HEAD) +BUILD_MACHINE=$(shell echo $$HOSTNAME) +BUILD_USER=$(shell whoami) +BUILD_ENVIRONMENT=$(BUILD_USER)@$(BUILD_MACHINE) + +BUILD_XDST=$(PACKAGE)/vendor/github.com/rebuy-de/rebuy-go-sdk/cmdutil +BUILD_FLAGS=-ldflags "\ + $(ADDITIONAL_LDFLAGS) \ + -X '$(BUILD_XDST).BuildName=$(NAME)' \ + -X '$(BUILD_XDST).BuildPackage=$(PACKAGE)' \ + -X '$(BUILD_XDST).BuildVersion=$(BUILD_VERSION)' \ + -X '$(BUILD_XDST).BuildDate=$(BUILD_DATE)' \ + -X '$(BUILD_XDST).BuildHash=$(BUILD_HASH)' \ + -X '$(BUILD_XDST).BuildEnvironment=$(BUILD_ENVIRONMENT)' \ +" + +GOFILES=$(shell find . -type f -name '*.go' -not -path "./vendor/*") +GOPKGS=$(shell go list ./...) + +default: build + +Gopkg.lock: Gopkg.toml + dep ensure + touch Gopkg.lock + +vendor: Gopkg.lock Gopkg.toml + dep ensure + touch vendor + +format: + gofmt -s -w $(GOFILES) + +vet: vendor + go vet $(GOPKGS) + +lint: + $(foreach pkg,$(GOPKGS),golint $(pkg);) + +test_gopath: + test $$(go list) = "$(PACKAGE)" + +test_packages: vendor + go test $(GOPKGS) + +test_format: + gofmt -s -l $(GOFILES) + +test: test_gopath test_format vet lint test_packages + +cov: + gocov test -v $(GOPKGS) \ + | gocov-html > coverage.html + +build: vendor + go build \ + $(BUILD_FLAGS) \ + -o $(NAME)-$(BUILD_VERSION)-$(shell go env GOOS)-$(shell go env GOARCH)$(shell go env GOEXE) + ln -sf $(NAME)-$(BUILD_VERSION)-$(shell go env GOOS)-$(shell go env GOARCH)$(shell go env GOEXE) $(NAME)$(shell go env GOEXE) + +xc: + GOOS=linux GOARCH=amd64 make build + GOOS=darwin GOARCH=amd64 make build + GOOS=windows GOARCH=386 make build + GOOS=windows GOARCH=amd64 make build + +install: test + go install \ + $(BUILD_FLAGS) + +clean: + rm -f $(NAME)* + +.PHONY: build install test diff --git a/main.go b/main.go new file mode 100644 index 0000000..01d2527 --- /dev/null +++ b/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/rebuy-de/cost-exporter/cmd" + "github.com/rebuy-de/rebuy-go-sdk/cmdutil" + + "github.com/sirupsen/logrus" +) + +func main() { + defer cmdutil.HandleExit() + if err := cmd.NewRootCommand().Execute(); err != nil { + logrus.Fatal(err) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..fbbd4b1 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,39 @@ +package config + +import ( + "io/ioutil" + + "github.com/sirupsen/logrus" + yaml "gopkg.in/yaml.v2" +) + +type Config struct { + Accounts []Account + Settings struct { + CostCron string `yaml:"costCron"` + CoresInterval int64 `yaml:"coresInterval"` + } +} + +type Account struct { + Name string + ID string + Secret string +} + +func Parse(configPath string) Config { + config := Config{} + var err error + + raw, err := ioutil.ReadFile(configPath) + if err != nil { + logrus.Fatal(err) + } + + err = yaml.Unmarshal([]byte(raw), &config) + if err != nil { + logrus.Fatal(err) + } + + return config +} diff --git a/pkg/prom/model.go b/pkg/prom/model.go new file mode 100644 index 0000000..018307f --- /dev/null +++ b/pkg/prom/model.go @@ -0,0 +1,61 @@ +package prom + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +type Costs struct { + Cost prometheus.GaugeVec + CoreCount prometheus.GaugeVec + ReservationCoverage prometheus.GaugeVec +} + +var C = Costs{ + Cost: *prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "rebuy", + Subsystem: "cost_exporter", + Name: "costs", + Help: "Costs by account and by service.", + }, + []string{"account", "service"}, + ), + CoreCount: *prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "rebuy", + Subsystem: "cost_exporter", + Name: "cores", + Help: "Count of all virtual CPUs in all regions of a specific account.", + }, + []string{"account", "region"}, + ), + ReservationCoverage: *prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "rebuy", + Subsystem: "cost_exporter", + Name: "reservationcoverage", + Help: "Coverage of running EC2 instances by reservations in percent.", + }, + []string{"account"}, + ), +} + +func (c *Costs) SetTotalCoreCount(account string, region string, count float64) { + c.CoreCount.With(prometheus.Labels{ + "account": account, + "region": region, + }).Set(count) +} + +func (c *Costs) SetCosts(account string, service string, cost float64) { + c.Cost.With(prometheus.Labels{ + "account": account, + "service": service, + }).Set(cost) +} + +func (c *Costs) SetReservationCoverage(account string, coverage float64) { + c.ReservationCoverage.With(prometheus.Labels{ + "account": account, + }).Set(coverage) +} diff --git a/pkg/prom/server.go b/pkg/prom/server.go new file mode 100644 index 0000000..47e5860 --- /dev/null +++ b/pkg/prom/server.go @@ -0,0 +1,23 @@ +package prom + +import ( + "fmt" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" +) + +func Run(port string) { + r := prometheus.NewRegistry() + r.MustRegister(C.CoreCount) + r.MustRegister(C.Cost) + r.MustRegister(C.ReservationCoverage) + + http.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{})) + + go func() { + logrus.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + }() +} diff --git a/pkg/retriever/cores.go b/pkg/retriever/cores.go new file mode 100644 index 0000000..21b942f --- /dev/null +++ b/pkg/retriever/cores.go @@ -0,0 +1,82 @@ +package retriever + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/rebuy-de/cost-exporter/pkg/config" + "github.com/rebuy-de/cost-exporter/pkg/prom" + "github.com/sirupsen/logrus" +) + +type CoreRetriever struct { + Accounts []config.Account + IntervalSec int64 + Services []Service +} + +type Service struct { + Region string + svc *ec2.EC2 + Account string +} + +func (c *CoreRetriever) Run() { + c.initialize() + c.getCores() + go c.scheduleInterval() +} + +func (c *CoreRetriever) initialize() { + for _, account := range c.Accounts { + regions := endpoints.AwsPartition().Services()[endpoints.Ec2ServiceID].Regions() + for regionName := range regions { + opts := session.Options{ + Config: aws.Config{ + Credentials: credentials.NewStaticCredentials( + account.ID, + account.Secret, + "", + )}} + sess := session.Must(session.NewSessionWithOptions(opts)) + svc := ec2.New(sess, aws.NewConfig().WithRegion(regionName)) + + c.Services = append(c.Services, Service{ + Region: regionName, + svc: svc, + Account: account.Name, + }) + } + } +} + +func (c *CoreRetriever) scheduleInterval() { + for range time.Tick(time.Duration(c.IntervalSec) * time.Second) { + c.getCores() + } +} + +func (c *CoreRetriever) getCores() { + for _, service := range c.Services { + logrus.Infof("Getting cores for account '%s' and region '%s'", service.Account, service.Region) + var totalCoreCount int64 + params := &ec2.DescribeInstancesInput{} + resp2, err := service.svc.DescribeInstances(params) + if err != nil { + fmt.Println(err) + } + for _, reservation := range resp2.Reservations { + for _, instance := range reservation.Instances { + if *instance.State.Name == "running" { + totalCoreCount = totalCoreCount + *instance.CpuOptions.CoreCount + } + } + } + prom.C.SetTotalCoreCount(service.Account, service.Region, float64(totalCoreCount)) + } +} diff --git a/pkg/retriever/cost.go b/pkg/retriever/cost.go new file mode 100644 index 0000000..847d830 --- /dev/null +++ b/pkg/retriever/cost.go @@ -0,0 +1,107 @@ +package retriever + +import ( + "strconv" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/costexplorer" + "github.com/rebuy-de/cost-exporter/pkg/config" + "github.com/rebuy-de/cost-exporter/pkg/prom" + "github.com/rebuy-de/cost-exporter/pkg/utils" + "github.com/robfig/cron" + "github.com/sirupsen/logrus" +) + +type CostRetriever struct { + Accounts []config.Account + Cron string + Services map[string]*costexplorer.CostExplorer +} + +func (c *CostRetriever) Run() { + c.initialize() + c.getCosts() + c.scheduleCron() +} + +func (c *CostRetriever) initialize() { + c.Services = make(map[string]*costexplorer.CostExplorer) + for _, account := range c.Accounts { + opts := session.Options{ + Config: aws.Config{ + Credentials: credentials.NewStaticCredentials( + account.ID, + account.Secret, + "", + )}} + sess := session.Must(session.NewSessionWithOptions(opts)) + svc := costexplorer.New(sess) + + c.Services[account.Name] = svc + } +} + +func (c *CostRetriever) scheduleCron() { + cron := cron.New() + cron.AddFunc(c.Cron, c.getCosts) + cron.Start() +} + +func (c *CostRetriever) getCosts() { + for _, account := range c.Accounts { + c.getReservationCoverage(account) + c.getCostsByService(account) + } +} + +func (c *CostRetriever) getCostsByService(account config.Account) { + logrus.Infof("Getting costs for account '%s'", account.Name) + svc := c.Services[account.Name] + respCost, err := svc.GetCostAndUsage((&costexplorer.GetCostAndUsageInput{ + Metrics: []*string{aws.String("BlendedCost")}, + // Getting the cost from 2 days ago is a workaround because getting data + // for yesterday yielded unstable numbers: + TimePeriod: utils.GetIntervalForPastDay(2), + Granularity: aws.String("DAILY"), + GroupBy: []*costexplorer.GroupDefinition{ + &costexplorer.GroupDefinition{ + Key: aws.String("SERVICE"), + Type: aws.String("DIMENSION"), + }, + }, + })) + if err != nil { + logrus.Fatal(err) + } + + for _, cost := range respCost.ResultsByTime[0].Groups { + amount, err := strconv.ParseFloat(*cost.Metrics["BlendedCost"].Amount, 64) + if err != nil { + logrus.Fatal(err) + } + prom.C.SetCosts(account.Name, *cost.Keys[0], amount) + } +} + +func (c *CostRetriever) getReservationCoverage(account config.Account) { + logrus.Infof("Getting reservation coverage for account '%s'", account.Name) + svc := c.Services[account.Name] + + respReservation, err := svc.GetReservationCoverage(&costexplorer.GetReservationCoverageInput{ + Granularity: aws.String("DAILY"), + // Unfortunately there is no newer data then 3 days ago. + TimePeriod: utils.GetIntervalForPastDay(3), + }) + if err != nil { + logrus.Fatal(err) + } + + coveragePercent, err := strconv.ParseFloat(*respReservation.Total.CoverageHours.CoverageHoursPercentage, 64) + if err != nil { + logrus.Fatal(err) + } + + prom.C.SetReservationCoverage(account.Name, coveragePercent) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000..fb5d7be --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,17 @@ +package utils + +import ( + "time" + + "github.com/aws/aws-sdk-go/service/costexplorer" +) + +func GetIntervalForPastDay(daysAgo int) *costexplorer.DateInterval { + now := time.Now() + start := now.AddDate(0, 0, -daysAgo) + end := now.AddDate(0, 0, (-daysAgo)+1) + dateRange := costexplorer.DateInterval{} + dateRange.SetStart(start.Format("2006-01-02")) + dateRange.SetEnd(end.Format("2006-01-02")) + return &dateRange +} diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..f99c419 --- /dev/null +++ b/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -x + +# This makes sure we don't spend a lot of money on API calls when the +# process starts crash looping for whatever reason: +./usr/local/bin/cost-exporter "$@" & + +while true; do sleep 365d; done