From 57dbf8c44c050eb0bbde060d27849abf475609a9 Mon Sep 17 00:00:00 2001 From: Alexandru Mihai Date: Wed, 22 Jan 2025 15:51:05 +0200 Subject: [PATCH] initial push --- .dockerignore | 20 ++ .envrc | 8 + .github/renovate.json | 52 +++++ .github/workflows/checks.yaml | 52 +++++ .github/workflows/ci.yaml | 52 +++++ .github/workflows/codeql.yaml | 44 ++++ .../workflows/container-registry-ghcr.yaml | 58 +++++ .gitignore | 3 + .golangci.yaml | 176 +++++++++++++++ .license-scan-overrides.jsonl | 7 + .license-scan-rules.json | 11 + Dockerfile | 36 +++ Makefile | 162 ++++++++++++++ Makefile.maker.yaml | 46 ++++ README.md | 58 ++++- REUSE.toml | 18 ++ cmd/main.go | 31 +++ config/config.go | 80 +++++++ go.mod | 20 ++ go.sum | 65 ++++++ internal/cmd/modules.go | 100 +++++++++ internal/cmd/providers.go | 167 ++++++++++++++ internal/cmd/root.go | 207 ++++++++++++++++++ shell.nix | 19 ++ 24 files changed, 1491 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .envrc create mode 100644 .github/renovate.json create mode 100644 .github/workflows/checks.yaml create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/codeql.yaml create mode 100644 .github/workflows/container-registry-ghcr.yaml create mode 100644 .gitignore create mode 100644 .golangci.yaml create mode 100644 .license-scan-overrides.jsonl create mode 100644 .license-scan-rules.json create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 Makefile.maker.yaml create mode 100644 REUSE.toml create mode 100644 cmd/main.go create mode 100644 config/config.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/cmd/modules.go create mode 100644 internal/cmd/providers.go create mode 100644 internal/cmd/root.go create mode 100644 shell.nix diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0656a22 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +/.dockerignore +.DS_Store +# TODO: uncomment when applications no longer use git to get version information +#.git/ +/.github/ +/.gitignore +/.goreleaser.yml +/*.env* +/.golangci.yaml +/.vscode/ +/build/ +/CONTRIBUTING.md +/Dockerfile +/docs/ +/LICENSE* +/Makefile.maker.yaml +/README.md +/report.html +/shell.nix +/testing/ diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..c8a98ce --- /dev/null +++ b/.envrc @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2019–2020 Target, Copyright 2021 The Nix Community +# SPDX-License-Identifier: Apache-2.0 +if type -P lorri &>/dev/null; then + eval "$(lorri direnv)" +else + use nix +fi diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 0000000..288ffca --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended", + "default:pinDigestsDisabled", + "mergeConfidence:all-badges", + "docker:disable" + ], + "assignees": [ + "defo89", + "SchwarzM", + "xsen84", + "goerangudat" + ], + "commitMessageAction": "Renovate: Update", + "constraints": { + "go": "1.23" + }, + "dependencyDashboardOSVVulnerabilitySummary": "all", + "osvVulnerabilityAlerts": true, + "postUpdateOptions": [ + "gomodTidy", + "gomodUpdateImportPaths" + ], + "packageRules": [ + { + "matchPackageNames": [ + "golang" + ], + "allowedVersions": "1.23.x" + }, + { + "matchPackageNames": [ + "/^github\\.com\\/sapcc\\/.*/" + ], + "automerge": true, + "groupName": "github.com/sapcc" + }, + { + "matchPackageNames": [ + "!/^github\\.com\\/sapcc\\/.*/", + "/.*/" + ], + "groupName": "External dependencies" + } + ], + "prHourlyLimit": 0, + "schedule": [ + "before 8am on Friday" + ], + "semanticCommits": "disabled" +} diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml new file mode 100644 index 0000000..1a19633 --- /dev/null +++ b/.github/workflows/checks.yaml @@ -0,0 +1,52 @@ +################################################################################ +# This file is AUTOGENERATED with # +# Edit Makefile.maker.yaml instead. # +################################################################################ + +# Copyright 2024 SAP SE +# SPDX-License-Identifier: Apache-2.0 + +name: Checks +"on": + push: + branches: + - main + pull_request: + branches: + - '*' + workflow_dispatch: {} +permissions: + checks: write + contents: read +jobs: + checks: + name: Checks + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + check-latest: true + go-version: 1.23.4 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: latest + - name: Dependency Licenses Review + run: make check-dependency-licenses + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + - name: Run govulncheck + run: govulncheck -format text ./... + - name: Check for spelling errors + uses: reviewdog/action-misspell@v1 + with: + exclude: ./vendor/* + fail_on_error: true + github_token: ${{ secrets.GITHUB_TOKEN }} + ignore: importas + reporter: github-check + - name: Check if source code files have license header + run: make check-license-headers diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..97b17a4 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,52 @@ +################################################################################ +# This file is AUTOGENERATED with # +# Edit Makefile.maker.yaml instead. # +################################################################################ + +# Copyright 2024 SAP SE +# SPDX-License-Identifier: Apache-2.0 + +name: CI +"on": + push: + branches: + - main + paths-ignore: + - '**.md' + pull_request: + branches: + - '*' + paths-ignore: + - '**.md' + workflow_dispatch: {} +permissions: + contents: read +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + check-latest: true + go-version: 1.23.4 + - name: Build all binaries + run: make build-all + test: + name: Test + needs: + - build + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + check-latest: true + go-version: 1.23.4 + - name: Run tests and generate coverage report + run: make build/cover.out diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 0000000..09e243c --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,44 @@ +################################################################################ +# This file is AUTOGENERATED with # +# Edit Makefile.maker.yaml instead. # +################################################################################ + +# Copyright 2024 SAP SE +# SPDX-License-Identifier: Apache-2.0 + +name: CodeQL +"on": + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '00 07 * * 1' + workflow_dispatch: {} +permissions: + actions: read + contents: read + security-events: write +jobs: + analyze: + name: CodeQL + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + check-latest: true + go-version: 1.23.4 + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: go + queries: security-extended + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/container-registry-ghcr.yaml b/.github/workflows/container-registry-ghcr.yaml new file mode 100644 index 0000000..720951f --- /dev/null +++ b/.github/workflows/container-registry-ghcr.yaml @@ -0,0 +1,58 @@ +################################################################################ +# This file is AUTOGENERATED with # +# Edit Makefile.maker.yaml instead. # +################################################################################ + +# Copyright 2024 SAP SE +# SPDX-License-Identifier: Apache-2.0 + +name: Container Registry GHCR +"on": + push: + branches: + - main + workflow_dispatch: {} +permissions: + contents: read + packages: write +jobs: + build-and-push-image: + name: Push container to ghcr.io + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + username: ${{ github.actor }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + # https://github.com/docker/metadata-action#typeedge + type=edge + # https://github.com/docker/metadata-action#latest-tag + type=raw,value=latest,enable={{is_default_branch}} + # https://github.com/docker/metadata-action#typesemver + type=semver,pattern={{raw}} + type=semver,pattern=v{{major}}.{{minor}} + type=semver,pattern=v{{major}} + # https://github.com/docker/metadata-action#typesha + type=sha,format=long + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a0fdb3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +certs/ +.Makefile +build/ \ No newline at end of file diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..35b091e --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,176 @@ +################################################################################ +# This file is AUTOGENERATED with # +# Edit Makefile.maker.yaml instead. # +################################################################################ + +# Copyright 2024 SAP SE +# SPDX-License-Identifier: Apache-2.0 + +run: + timeout: 3m # 1m by default + modules-download-mode: readonly + +output: + # Do not print lines of code with issue. + print-issued-lines: false + +issues: + exclude: + # It is idiomatic Go to reuse the name 'err' with ':=' for subsequent errors. + # Ref: https://go.dev/doc/effective_go#redeclaration + - 'declaration of "err" shadows declaration at' + exclude-rules: + - path: _test\.go + linters: + - bodyclose + - dupl + # '0' disables the following options. + max-issues-per-linter: 0 + max-same-issues: 0 + +linters-settings: + dupl: + # Tokens count to trigger issue, 150 by default. + threshold: 100 + errcheck: + # Report about assignment of errors to blank identifier. + check-blank: true + # Do not report about not checking of errors in type assertions. + # This is not as dangerous as skipping error values because an unchecked type assertion just immediately panics. + # We disable this because it makes a ton of useless noise esp. in test code. + check-type-assertions: false + forbidigo: + analyze-types: true # required for pkg: + forbid: + # ioutil package has been deprecated: https://github.com/golang/go/issues/42026 + - ^ioutil\..*$ + # Using http.DefaultServeMux is discouraged because it's a global variable that some packages silently and magically add handlers to (esp. net/http/pprof). + # Applications wishing to use http.ServeMux should obtain local instances through http.NewServeMux() instead of using the global default instance. + - ^http\.DefaultServeMux$ + - ^http\.Handle(?:Func)?$ + # Forbid usage of old and archived square/go-jose + - pkg: ^gopkg\.in/square/go-jose\.v2$ + msg: "gopk.in/square/go-jose is archived and has CVEs. Replace it with gopkg.in/go-jose/go-jose.v2" + - pkg: ^github.com/coreos/go-oidc$ + msg: "github.com/coreos/go-oidc depends on gopkg.in/square/go-jose which has CVEs. Replace it with github.com/coreos/go-oidc/v3" + + - pkg: ^github.com/howeyc/gopass$ + msg: "github.com/howeyc/gopass is archived, use golang.org/x/term instead" + goconst: + ignore-tests: true + min-occurrences: 5 + gocritic: + enabled-checks: + - boolExprSimplify + - builtinShadow + - emptyStringTest + - evalOrder + - httpNoBody + - importShadow + - initClause + - methodExprCall + - paramTypeCombine + - preferFilepathJoin + - ptrToRefParam + - redundantSprint + - returnAfterHttpError + - stringConcatSimplify + - timeExprSimplify + - truncateCmp + - typeAssertChain + - typeUnparen + - unnamedResult + - unnecessaryBlock + - unnecessaryDefer + - weakCond + - yodaStyleExpr + goimports: + # Put local imports after 3rd-party packages. + local-prefixes: github.com/sapcc/tf-registry + gomoddirectives: + go-version-pattern: '1\.\d+(\.0)?$' + replace-allow-list: + # for go-pmtud + - github.com/mdlayher/arp + toolchain-forbidden: true + gosec: + excludes: + # gosec wants us to set a short ReadHeaderTimeout to avoid Slowloris attacks, but doing so would expose us to Keep-Alive race conditions (see https://iximiuz.com/en/posts/reverse-proxy-http-keep-alive-and-502s/) + - G112 + # created file permissions are restricted by umask if necessary + - G306 + govet: + enable-all: true + disable: + - fieldalignment + nolintlint: + require-specific: true + stylecheck: + dot-import-whitelist: + - github.com/onsi/ginkgo/v2 + - github.com/onsi/gomega + usestdlibvars: + constant-kind: true + crypto-hash: true + default-rpc-path: true + http-method: true + http-status-code: true + sql-isolation-level: true + time-layout: true + time-month: true + time-weekday: true + tls-signature-scheme: true + usetesting: + os-setenv: true + os-temp-dir: true + whitespace: + # Enforce newlines (or comments) after multi-line function signatures. + multi-func: true + +linters: + # We use 'disable-all' and enable linters explicitly so that a newer version + # does not introduce new linters unexpectedly. + disable-all: true + enable: + - bodyclose + - containedctx + - copyloopvar + - dupl + - dupword + - durationcheck + - errcheck + - errname + - errorlint + - exptostd + - forbidigo + - ginkgolinter + - gocheckcompilerdirectives + - goconst + - gocritic + - gofmt + - goimports + - gomoddirectives + - gosec + - gosimple + - govet + - ineffassign + - intrange + - misspell + - nilerr + - noctx + - nolintlint + - nosprintfhostport + - perfsprint + - predeclared + - rowserrcheck + - sqlclosecheck + - staticcheck + - stylecheck + - tenv + - typecheck + - unconvert + - unparam + - unused + - usestdlibvars + - usetesting + - whitespace diff --git a/.license-scan-overrides.jsonl b/.license-scan-overrides.jsonl new file mode 100644 index 0000000..29aac5c --- /dev/null +++ b/.license-scan-overrides.jsonl @@ -0,0 +1,7 @@ +{"name": "github.com/chzyer/logex", "licenceType": "MIT"} +{"name": "github.com/hashicorp/vault/api/auth/approle", "licenceType": "MPL-2.0"} +{"name": "github.com/jpillora/longestcommon", "licenceType": "MIT"} +{"name": "github.com/spdx/tools-golang", "licenceTextOverrideFile": "vendor/github.com/spdx/tools-golang/LICENSE.code"} +{"name": "github.com/xeipuuv/gojsonpointer", "licenceType": "Apache-2.0"} +{"name": "github.com/xeipuuv/gojsonreference", "licenceType": "Apache-2.0"} +{"name": "github.com/xeipuuv/gojsonschema", "licenceType": "Apache-2.0"} diff --git a/.license-scan-rules.json b/.license-scan-rules.json new file mode 100644 index 0000000..e584e9d --- /dev/null +++ b/.license-scan-rules.json @@ -0,0 +1,11 @@ +{ + "allowlist": [ + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "MIT", + "MPL-2.0", + "Unlicense" + ] +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f6a8e7a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +FROM golang:1.23.4-alpine3.21 AS builder + +RUN apk add --no-cache --no-progress ca-certificates gcc git make musl-dev + +COPY . /src +ARG BININFO_BUILD_DATE BININFO_COMMIT_HASH BININFO_VERSION # provided to 'make install' +RUN make -C /src install PREFIX=/pkg GOTOOLCHAIN=local + +################################################################################ + +FROM alpine:3.21 + +RUN addgroup -g 4200 appgroup \ + && adduser -h /home/appuser -s /sbin/nologin -G appgroup -D -u 4200 appuser + +# upgrade all installed packages to fix potential CVEs in advance +# also remove apk package manager to hopefully remove dependency on OpenSSL 🤞 +RUN apk upgrade --no-cache --no-progress \ + && apk del --no-cache --no-progress apk-tools alpine-keys + +COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs/ +COPY --from=builder /etc/ssl/cert.pem /etc/ssl/cert.pem +COPY --from=builder /pkg/ /usr/ +# make sure all binaries can be executed +RUN tf-registry --version 2>/dev/null + +ARG BININFO_BUILD_DATE BININFO_COMMIT_HASH BININFO_VERSION +LABEL source_repository="https://github.com/sapcc/argora" \ + org.opencontainers.image.url="https://github.com/sapcc/argora" \ + org.opencontainers.image.created=${BININFO_BUILD_DATE} \ + org.opencontainers.image.revision=${BININFO_COMMIT_HASH} \ + org.opencontainers.image.version=${BININFO_VERSION} + +USER 4200:4200 +WORKDIR /home/appuser +ENTRYPOINT [ "/usr/bin/tf-registry" ] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2bd63fc --- /dev/null +++ b/Makefile @@ -0,0 +1,162 @@ +################################################################################ +# This file is AUTOGENERATED with # +# Edit Makefile.maker.yaml instead. # +################################################################################ + +# Copyright 2024 SAP SE +# SPDX-License-Identifier: Apache-2.0 + +MAKEFLAGS=--warn-undefined-variables +# /bin/sh is dash on Debian which does not support all features of ash/bash +# to fix that we use /bin/bash only on Debian to not break Alpine +ifneq (,$(wildcard /etc/os-release)) # check file existence + ifneq ($(shell grep -c debian /etc/os-release),0) + SHELL := /bin/bash + endif +endif + +default: build-all + +tilt: FORCE generate + +install-golangci-lint: FORCE + @if ! hash golangci-lint 2>/dev/null; then printf "\e[1;36m>> Installing golangci-lint (this may take a while)...\e[0m\n"; go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; fi + +install-go-licence-detector: FORCE + @if ! hash go-licence-detector 2>/dev/null; then printf "\e[1;36m>> Installing go-licence-detector...\e[0m\n"; go install go.elastic.co/go-licence-detector@latest; fi + +install-addlicense: FORCE + @if ! hash addlicense 2>/dev/null; then printf "\e[1;36m>> Installing addlicense...\e[0m\n"; go install github.com/google/addlicense@latest; fi + +prepare-static-check: FORCE install-golangci-lint install-go-licence-detector install-addlicense + +GO_BUILDFLAGS = +GO_LDFLAGS = +GO_TESTENV = +GO_BUILDENV = + +# These definitions are overridable, e.g. to provide fixed version/commit values when +# no .git directory is present or to provide a fixed build date for reproducibility. +BININFO_VERSION ?= $(shell git describe --tags --always --abbrev=7) +BININFO_COMMIT_HASH ?= $(shell git rev-parse --verify HEAD) +BININFO_BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") + +build-all: build/tf-registry + +build/tf-registry: FORCE + @env $(GO_BUILDENV) go build $(GO_BUILDFLAGS) -ldflags '-s -w -X github.com/sapcc/go-api-declarations/bininfo.binName=tf-registry -X github.com/sapcc/go-api-declarations/bininfo.version=$(BININFO_VERSION) -X github.com/sapcc/go-api-declarations/bininfo.commit=$(BININFO_COMMIT_HASH) -X github.com/sapcc/go-api-declarations/bininfo.buildDate=$(BININFO_BUILD_DATE) $(GO_LDFLAGS)' -o build/tf-registry ./cmd/ + +DESTDIR = +ifeq ($(shell uname -s),Darwin) + PREFIX = /usr/local +else + PREFIX = /usr +endif + +install: FORCE build/tf-registry + install -d -m 0755 "$(DESTDIR)$(PREFIX)/bin" + install -m 0755 build/tf-registry "$(DESTDIR)$(PREFIX)/bin/tf-registry" + +# which packages to test with test runner +GO_TESTPKGS := $(shell go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./...) +ifeq ($(GO_TESTPKGS),) +GO_TESTPKGS := ./... +endif +# which packages to measure coverage for +GO_COVERPKGS := $(shell go list ./...) +# to get around weird Makefile syntax restrictions, we need variables containing nothing, a space and comma +null := +space := $(null) $(null) +comma := , + +check: FORCE static-check build/cover.html build-all + @printf "\e[1;32m>> All checks successful.\e[0m\n" + +run-golangci-lint: FORCE install-golangci-lint + @printf "\e[1;36m>> golangci-lint\e[0m\n" + @golangci-lint run + +build/cover.out: FORCE | build + @printf "\e[1;36m>> Running tests\e[0m\n" + @env $(GO_TESTENV) go test -shuffle=on -p 1 -coverprofile=$@ $(GO_BUILDFLAGS) -ldflags '-s -w -X github.com/sapcc/go-api-declarations/bininfo.binName=tf-registry -X github.com/sapcc/go-api-declarations/bininfo.version=$(BININFO_VERSION) -X github.com/sapcc/go-api-declarations/bininfo.commit=$(BININFO_COMMIT_HASH) -X github.com/sapcc/go-api-declarations/bininfo.buildDate=$(BININFO_BUILD_DATE) $(GO_LDFLAGS)' -covermode=count -coverpkg=$(subst $(space),$(comma),$(GO_COVERPKGS)) $(GO_TESTPKGS) + +build/cover.html: build/cover.out + @printf "\e[1;36m>> go tool cover > build/cover.html\e[0m\n" + @go tool cover -html $< -o $@ + +static-check: FORCE run-golangci-lint check-dependency-licenses check-license-headers + +build: + @mkdir $@ + +tidy-deps: FORCE + go mod tidy + go mod verify + +force-license-headers: FORCE install-addlicense + @printf "\e[1;36m>> addlicense\e[0m\n" + echo -n $(patsubst $(shell awk '$$1 == "module" {print $$2}' go.mod)%,.%/*.go,$(shell go list ./...)) | xargs -d" " -I{} bash -c 'year="$$(rg -P "Copyright (....) SAP SE" -Nor "\$$1" {})"; awk -i inplace '"'"'{if (display) {print} else {!/^\/\*/ && !/^\*/ && !/^\$$/}}; /^package /{print;display=1}'"'"' {}; addlicense -c "SAP SE" -s=only -y "$$year" -- {}' + +license-headers: FORCE install-addlicense + @printf "\e[1;36m>> addlicense\e[0m\n" + @addlicense -c "SAP SE" -s=only -- $(patsubst $(shell awk '$$1 == "module" {print $$2}' go.mod)%,.%/*.go,$(shell go list ./...)) + +check-license-headers: FORCE install-addlicense + @printf "\e[1;36m>> addlicense --check\e[0m\n" + @addlicense --check -- $(patsubst $(shell awk '$$1 == "module" {print $$2}' go.mod)%,.%/*.go,$(shell go list ./...)) + +check-dependency-licenses: FORCE install-go-licence-detector + @printf "\e[1;36m>> go-licence-detector\e[0m\n" + @go list -m -mod=readonly -json all | go-licence-detector -includeIndirect -rules .license-scan-rules.json -overrides .license-scan-overrides.jsonl + +clean: FORCE + git clean -dxf build + +vars: FORCE + @printf "BININFO_BUILD_DATE=$(BININFO_BUILD_DATE)\n" + @printf "BININFO_COMMIT_HASH=$(BININFO_COMMIT_HASH)\n" + @printf "BININFO_VERSION=$(BININFO_VERSION)\n" + @printf "DESTDIR=$(DESTDIR)\n" + @printf "GO_BUILDENV=$(GO_BUILDENV)\n" + @printf "GO_BUILDFLAGS=$(GO_BUILDFLAGS)\n" + @printf "GO_COVERPKGS=$(GO_COVERPKGS)\n" + @printf "GO_LDFLAGS=$(GO_LDFLAGS)\n" + @printf "GO_TESTENV=$(GO_TESTENV)\n" + @printf "GO_TESTPKGS=$(GO_TESTPKGS)\n" + @printf "PREFIX=$(PREFIX)\n" +help: FORCE + @printf "\n" + @printf "\e[1mUsage:\e[0m\n" + @printf " make \e[36m\e[0m\n" + @printf "\n" + @printf "\e[1mGeneral\e[0m\n" + @printf " \e[36mvars\e[0m Display values of relevant Makefile variables.\n" + @printf " \e[36mhelp\e[0m Display this help.\n" + @printf "\n" + @printf "\e[1mPrepare\e[0m\n" + @printf " \e[36minstall-golangci-lint\e[0m Install golangci-lint required by run-golangci-lint/static-check\n" + @printf " \e[36minstall-go-licence-detector\e[0m Install-go-licence-detector required by check-dependency-licenses/static-check\n" + @printf " \e[36minstall-addlicense\e[0m Install addlicense required by check-license-headers/license-headers/static-check\n" + @printf " \e[36mprepare-static-check\e[0m Install any tools required by static-check. This is used in CI before dropping privileges, you should probably install all the tools using your package manager\n" + @printf "\n" + @printf "\e[1mBuild\e[0m\n" + @printf " \e[36mbuild-all\e[0m Build all binaries.\n" + @printf " \e[36mbuild/tf-registry\e[0m Build tf-registry.\n" + @printf " \e[36minstall\e[0m Install all binaries. This option understands the conventional 'DESTDIR' and 'PREFIX' environment variables for choosing install locations.\n" + @printf "\n" + @printf "\e[1mTest\e[0m\n" + @printf " \e[36mcheck\e[0m Run the test suite (unit tests and golangci-lint).\n" + @printf " \e[36mrun-golangci-lint\e[0m Install and run golangci-lint. Installing is used in CI, but you should probably install golangci-lint using your package manager.\n" + @printf " \e[36mbuild/cover.out\e[0m Run tests and generate coverage report.\n" + @printf " \e[36mbuild/cover.html\e[0m Generate an HTML file with source code annotations from the coverage report.\n" + @printf " \e[36mstatic-check\e[0m Run static code checks\n" + @printf "\n" + @printf "\e[1mDevelopment\e[0m\n" + @printf " \e[36mtidy-deps\e[0m Run go mod tidy and go mod verify.\n" + @printf " \e[36mforce-license-headers\e[0m Remove and re-add all license headers to all non-vendored source code files.\n" + @printf " \e[36mlicense-headers\e[0m Add license headers to all non-vendored source code files.\n" + @printf " \e[36mcheck-license-headers\e[0m Check license headers in all non-vendored .go files.\n" + @printf " \e[36mcheck-dependency-licenses\e[0m Check all dependency licenses using go-licence-detector.\n" + @printf " \e[36mclean\e[0m Run git clean.\n" + +.PHONY: FORCE diff --git a/Makefile.maker.yaml b/Makefile.maker.yaml new file mode 100644 index 0000000..4e313dd --- /dev/null +++ b/Makefile.maker.yaml @@ -0,0 +1,46 @@ +# Configuration file for + +metadata: + url: https://github.com/sapcc/argora + +binaries: + - name: tf-registry + fromPackage: ./cmd/ + installTo: bin/ + +golang: + setGoModVersion: true + +golangciLint: + createConfig: true + +githubWorkflow: + ci: + enabled: true + coveralls: false + ignorePaths: + - "**.md" # all Markdown files + pushContainerToGhcr: + enabled: true + platforms: "linux/amd64" + tagStrategy: + - edge + - latest + - semver + - sha + +renovate: + enabled: true + assignees: + - defo89 + - SchwarzM + - xsen84 + - goerangudat + +dockerfile: + enabled: true + + +verbatim: | + tilt: FORCE generate + tilt up --stream -- --BININFO_VERSION $(BININFO_VERSION) --BININFO_COMMIT_HASH $(BININFO_COMMIT_HASH) --BININFO_BUILD_DATE $(BININFO_BUILD_DATE) \ No newline at end of file diff --git a/README.md b/README.md index 9888ba4..ef90203 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,58 @@ # tf-registry -a terraform private registry +Self Hosted Terraform Registry backed by S3 +Based on the initial work of [terraform-registry](https://github.com/nrkno/terraform-registry) + +## Usage + + +`tf-registry` Provides a simple http server that implements the [Terraform Provider Registry Protocol](https://developer.hashicorp.com/terraform/internals/provider-registry-protocol). + +``` +Terraform Registry Server + +Usage: tf-registry [flags] + +Flags: + -bucket string + aws s3 bucket name containing terraform providers + -port string + port for HTTPS server (default "443") + -profile string + aws named profile to assume (default "default") + - serverCert + path to https server certificate + - serverKey + path to https server key + - gpgKey + path to gpg public key +``` + +### Uploading Providers + +Pre requisites: +A GPG key pair was created and stored in vault. + +Uploading is done via: https://ci1.eu-de-2.cloud.sap/teams/services/pipelines/terraform-providers +Process is explained below: +1. build new provider from source. name must be terraform-provider- +2. zip it to terraform-provide-___.zip +3. get the sha256 sum of the file and put it in the SHA256SUM file: shasum -a 256 >> SHA256SUM +4. get the detached signature file: gpg -b SHA256SUM +5. upload the provider zip file to s3: bucket/org/name/version/provider.zip. e.g. terraform-registry-1/cp/daybreak/1.0.0/terraform-provider-daybreak_1.0.0_linux_amd64.zip +6. upload SHA256SUM and SHA256SUM.SIG to s3: bucket/org/name/version/, e.g. terraform-registry-1/cp/daybreak/1.0.0/signatures + +### Starting server +Server is deployed via pipeline: +https://ci1.eu-de-2.cloud.sap/teams/services/pipelines/terraform-registry +which deploys this chart: +https://github.wdf.sap.corp/cc/terraform-registry + +E.g: start the server: ./tf-registry -bucket terraform-registry-1 -port 443 -serverCert certs/localhost.crt -serverKey certs/localhost.key -gpgKey certs/gpg.pub +Certs and key will be mounted to /certs, gpg key via /gpg/gpg.pub + +## TODO + +- manage providers versioning + + + diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 0000000..552d440 --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: SAP SE +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[annotations]] +path = [ + ".github/CODEOWNERS", + ".github/renovate.json", + ".gitignore", + ".license-scan-overrides.jsonl", + ".license-scan-rules.json", + "go.mod", + "go.sum", + "Makefile.maker.yaml", +] +SPDX-FileCopyrightText = "SAP SE" +SPDX-License-Identifier = "Apache-2.0" diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..5cd960c --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,31 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "github.com/sapcc/tf-registry/internal/cmd" +) + +func main() { + err := cmd.RootCmd.Execute() + if err != nil { + println(err.Error()) + os.Exit(1) + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..056b804 --- /dev/null +++ b/config/config.go @@ -0,0 +1,80 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import "io/fs" + +const ProviderBasePath = "/providers" +const ModuleBasePath = "/modules" + +type ServiceDiscoveryResp struct { + ProvidersV1 string `json:"providers.v1"` + ModulesV1 string `json:"modules.v1"` +} + +type ProviderVersions struct { + Versions []map[string]string `json:"versions"` +} + +type Provider struct { + Namespace string + Provider string + Version string + Os string + Arch string +} + +type ProviderResp struct { + DownloadURL string `json:"download_url"` + Shasum string `json:"shasum"` + Os string `json:"os"` + Arch string `json:"arch"` + Filename string `json:"filename"` + ShasumsURL string `json:"shasums_url"` + ShasumsSignatureURL string `json:"shasums_signature_url"` + SigningKeys map[string]interface{} `json:"signing_keys"` +} + +type ModuleVersions struct { + Versions []map[string]string `json:"versions"` +} + +// ModuleVersionsResp is our module versions response struct +type ModuleVersionsResp struct { + Modules []ModuleVersions `json:"modules"` +} + +// Module respresents a terraform module +type Module struct { + Namespace string + Name string + Provider string + Version string +} + +// Commandline parameters +type CmdLineParams struct { + ServerCert string // https certificate + ServerKey string // https key + GpgKey string // gpg public key location + Pprefix string + Mprefix string + Port string + + Bucket string + S3fsys fs.FS +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9125eef --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module github.com/sapcc/tf-registry + +go 1.23 + +require ( + github.com/aws/aws-sdk-go v1.44.139 + github.com/go-chi/chi/v5 v5.0.7 + github.com/jszwec/s3fs v0.4.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) + +require ( + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/sapcc/go-api-declarations v1.13.2 + github.com/spf13/cobra v1.8.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..336f105 --- /dev/null +++ b/go.sum @@ -0,0 +1,65 @@ +github.com/aws/aws-sdk-go v1.36.24/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.139 h1:Mj/OZBy9RTbzJ8pfgK6rOL8xgUEAIn8pfIN6qWFtpAk= +github.com/aws/aws-sdk-go v1.44.139/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= +github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jszwec/s3fs v0.4.0 h1:8FlE71TTzxykwh+CAZtASNVKHHw6LO43VlsjP60r+Ic= +github.com/jszwec/s3fs v0.4.0/go.mod h1:+FmWmocDLzba/O3eTTc2MXb1a3O8vkoul2C/Cm2lNOc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sapcc/go-api-declarations v1.13.2 h1:dPYYsjwKGObSAm6+K+dYCiLQWunYuWkywlZnuXfjsmk= +github.com/sapcc/go-api-declarations v1.13.2/go.mod h1:83R3hTANhuRXt/pXDby37IJetw8l7DG41s33Tp9NXxI= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cmd/modules.go b/internal/cmd/modules.go new file mode 100644 index 0000000..b60b125 --- /dev/null +++ b/internal/cmd/modules.go @@ -0,0 +1,100 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "encoding/json" + "fmt" + "io/fs" + "net/http" + "path/filepath" + + "github.com/sapcc/tf-registry/config" + + "github.com/go-chi/chi/v5" +) + +func httpGetModuleVersions(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetModuleVersions") + m := config.Module{ + Namespace: chi.URLParam(r, "namespace"), + Name: chi.URLParam(r, "name"), + Provider: chi.URLParam(r, "provider"), + } + modPath := filepath.Join(C.Mprefix, m.Namespace, m.Name, m.Provider) + fmt.Println(" > modpath:", modPath) + modVers, err := getModuleVersions(modPath) + if err != nil { + // TODO handle module not found with 404 + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(modVers) + if err != nil { + fmt.Println(err) + } +} + +func getModuleVersions(modPath string) (config.ModuleVersionsResp, error) { + fmt.Println(" > inside getModuleVersions") + fmt.Println(" >> modpath:", modPath) + + m := config.ModuleVersions{} + versionDirs, err := fs.ReadDir(C.S3fsys, modPath) + fmt.Println(" >> versionDirs:", versionDirs) + if err != nil { + return config.ModuleVersionsResp{}, err + } + for _, v := range versionDirs { + vers := map[string]string{"version": v.Name()} + m.Versions = append(m.Versions, vers) + fmt.Println(" >> found version: ", vers) + } + return config.ModuleVersionsResp{ + Modules: []config.ModuleVersions{m}, + }, nil +} + +func httpGetModuleDownloadURL(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetModuleDownloadURL") + m := config.Module{ + Namespace: chi.URLParam(r, "namespace"), + Name: chi.URLParam(r, "name"), + Provider: chi.URLParam(r, "provider"), + Version: chi.URLParam(r, "version"), + } + tfGetHeader := filepath.Join( + "/download/modules", + m.Namespace, + m.Name, + m.Provider, + m.Version, + m.Name+".tgz", + ) + fmt.Println(" > tfGetHeader:", tfGetHeader) + w.Header().Set("X-Terraform-Get", tfGetHeader) + w.WriteHeader(http.StatusNoContent) +} + +func httpGetModule(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetModule") + w.Header().Set("Content-Encoding", "application/octet-stream") + w.Header().Set("Content-Type", "application/x-gzip") + fs := http.StripPrefix("/download/", http.FileServer(http.FS(C.S3fsys))) + fs.ServeHTTP(w, r) +} diff --git a/internal/cmd/providers.go b/internal/cmd/providers.go new file mode 100644 index 0000000..3910dc6 --- /dev/null +++ b/internal/cmd/providers.go @@ -0,0 +1,167 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/fs" + "log" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/go-chi/chi/v5" + + "github.com/sapcc/tf-registry/config" +) + +func getProviderVersions(provPath string) (config.ProviderVersions, error) { + fmt.Println(" > inside getProviderVersions") + fmt.Println(" >> provPath:", provPath) + p := config.ProviderVersions{} + versionDirs, err := fs.ReadDir(C.S3fsys, provPath) + fmt.Println(" >> versionDirs:", versionDirs) + if err != nil { + return config.ProviderVersions{}, err + } + for _, v := range versionDirs { + vers := map[string]string{"version": v.Name()} + p.Versions = append(p.Versions, vers) + fmt.Println(" >> found version: ", vers) + } + return p, nil +} + +// httpGetVersions is a http handler for retrieving a list of module versions +// the registry server expects the versions to all be a set of +func httpGetProviderVersions(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetProviderVersions") + p := config.Provider{ + Namespace: chi.URLParam(r, "namespace"), + Provider: chi.URLParam(r, "provider"), + } + + provPath := filepath.Join(C.Pprefix, p.Namespace, p.Provider) + fmt.Println(" >> provPath: ", provPath) + + provVers, err := getProviderVersions(provPath) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(provVers) + if err != nil { + fmt.Println(err) + } +} + +// function to get the shasum for a specific provider +func getSHASUM(downloadPath string) string { + fmt.Println(" > inside getSHASUM") + s3file := strings.Trim(downloadPath, "downla/") + fmt.Println(" >> file to be open: ", s3file) + f, err := C.S3fsys.Open(s3file) + if err != nil { + fmt.Println(err) + } + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + log.Fatal(err) + } + sha256sum := hex.EncodeToString(h.Sum(nil)) + fmt.Println(" >> calculated sum: ", sha256sum) + + return sha256sum +} + +// function to read public GPG key +func getGPGkey(keyFile string) string { + fmt.Println(" > inside getGPGL", keyFile) + content, err := os.ReadFile(keyFile) + if err != nil { + fmt.Println(err) + } + fmt.Println(" >> content (first 10 chars): ", content[0:10]) + return string(content) +} + +// httpGetProviderDownLoadURL is a http handler for retrieving the final download URL for a terraform module, +func httpGetProviderDownloadURL(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetProviderDownloadURL") + fmt.Println(" >> building json response") + + p := config.Provider{ + Namespace: chi.URLParam(r, "namespace"), + Provider: chi.URLParam(r, "provider"), + Version: chi.URLParam(r, "version"), + Os: chi.URLParam(r, "os"), + Arch: chi.URLParam(r, "arch"), + } + downloadPrefix := filepath.Join("/download/", C.Pprefix, "/", p.Namespace, p.Provider, p.Version) + signaturePrefix := filepath.Join("/", C.Pprefix, "/", p.Namespace, p.Provider, p.Version, "/signatures") + filename := "terraform-provider-" + p.Provider + "_" + p.Version + "_" + p.Os + "_" + p.Arch + ".zip" + fmt.Printf(" >> see complete json here: https:///%s/%s/%s/download/%s/%s", p.Namespace, p.Provider, p.Version, p.Os, p.Arch) + + // building response json: https://developer.hashicorp.com/terraform/internals/provider-registry-protocol#protocols-1 + + providerResponse := config.ProviderResp{} + providerResponse.DownloadURL = downloadPrefix + "/" + filename + fmt.Println(" >> downloadURL: ", providerResponse.DownloadURL) + providerResponse.Filename = filename + providerResponse.Shasum = getSHASUM(providerResponse.DownloadURL) // calculated ar GET time + providerResponse.ShasumsURL = signaturePrefix + "/SHA256SUMS" + providerResponse.Os = p.Os + providerResponse.Arch = p.Arch + providerResponse.ShasumsSignatureURL = signaturePrefix + "/SHA256SUMS.sig" + providerResponse.SigningKeys = map[string]interface{}{"gpg_public_keys": []map[string]interface{}{{"key_id": "terraform-registry", "ascii_armor": getGPGkey(C.GpgKey)}}} + + // fields that the documentation list as required but does not seem to be needed for now + // providerResponse.Protocols = []string{"4.0", "5.1"} + // providerResponse.Trust_signature = "" + // providerResponse.Source = "some org name" + // providerResponse.Source_url = "/" + + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(providerResponse) + if err != nil { + fmt.Println(err) + } +} + +// http handler for retrieving a terraform provider + +func httpGetProvider(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetProvider") + w.Header().Set("Content-Encoding", "application/octet-stream") + w.Header().Set("Content-Type", "application/x-gzip") + fs := http.StripPrefix("/download/", http.FileServer(http.FS(C.S3fsys))) + fs.ServeHTTP(w, r) +} + +func httpGetSignatures(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetSignatures") + w.Header().Set("Content-Type", "text") + fs := http.FileServer(http.FS(C.S3fsys)) + fs.ServeHTTP(w, r) +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go new file mode 100644 index 0000000..da2f0e1 --- /dev/null +++ b/internal/cmd/root.go @@ -0,0 +1,207 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "encoding/json" + "fmt" + "io/fs" + "net/http" + "os" + "time" + + "github.com/sapcc/tf-registry/config" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/jszwec/s3fs" + "github.com/sapcc/go-api-declarations/bininfo" + "github.com/spf13/cobra" +) + +// Globals + +var C = &config.CmdLineParams{} + +var RootCmd = &cobra.Command{ + Use: "tf-registry", + Short: "tf-registry is a private terraform registry witha s3 backend", + RunE: RunRootCmd, + Version: bininfo.Version(), +} + +func init() { + RootCmd.PersistentFlags().StringVar(&C.ServerCert, "serverCert", "", "path to https server certificate") + RootCmd.PersistentFlags().StringVar(&C.ServerKey, "serverKey", "", "path to https server key") + RootCmd.PersistentFlags().StringVar(&C.GpgKey, "gpgKey", "", "path to gpg public key") + RootCmd.PersistentFlags().StringVar(&C.Pprefix, "pprefix", "", "optional path prefix for local providers") + RootCmd.PersistentFlags().StringVar(&C.Mprefix, "mprefix", "", "optional path prefix for local modules") + RootCmd.PersistentFlags().StringVar(&C.Port, "port", "443", "port for HTTP server") + RootCmd.PersistentFlags().StringVar(&C.Bucket, "bucket", "", "aws s3 bucket name containing terraform providers") + err := RootCmd.MarkPersistentFlagRequired("serverCert") + if err != nil { + fmt.Println(err) + } + err = RootCmd.MarkPersistentFlagRequired("serverKey") + if err != nil { + fmt.Println(err) + } + err = RootCmd.MarkPersistentFlagRequired("gpgKey") + if err != nil { + fmt.Println(err) + } + err = RootCmd.MarkPersistentFlagRequired("pprefix") + if err != nil { + fmt.Println(err) + } + err = RootCmd.MarkPersistentFlagRequired("mprefix") + if err != nil { + fmt.Println(err) + } + err = RootCmd.MarkPersistentFlagRequired("bucket") + if err != nil { + fmt.Println(err) + } +} + +// TF Registry Server +func RunRootCmd(cmd *cobra.Command, args []string) error { + fmt.Printf(" > Starting tf-registry webserver on 0.0.0.0:%s...\n", C.Port) + fmt.Println(" > Connecting to storage backend...") + + fmt.Println(" > Using env vars to connect to aws") + + sess := session.Must(session.NewSession()) + + C.S3fsys = s3fs.New(s3.New(sess), C.Bucket) + bucketRoot := "." + fmt.Println(" > bucketroot:", bucketRoot) + stat, errr := fs.Stat(C.S3fsys, bucketRoot) + if errr != nil { + fmt.Println(errr) + os.Exit(1) + } + + fmt.Println(" > fsstat:", stat) + + fmt.Printf(" > Connection successful, serving terraform registry from: s3://%s/\n", C.Bucket) + + // Configure a go-chi router + r := chi.NewRouter() + r.Use(middleware.RealIP) + r.Use(middleware.RequestID) + r.Use(middleware.Recoverer) + r.Use(middleware.Logger) + r.Use(middleware.GetHead) + + // is http server alive? + r.Get("/alive", httpIAmAlive) + + // is s3 connection working? + r.Get("/s3_reachable", httpS3Status) + // ROUTES + + // GET / returns our static service discovery resp + r.Get("/", httpGetServiceDiscovery) + + // GET /.well-known/terraform.json returns our static service discovery resp + r.Get("/.well-known/terraform.json", httpGetServiceDiscovery) + + // PROVIDERS + + // GET /:namespace/:name/:provider/versions returns a list of versions for the specified module path + r.Get("/{namespace}/{provider}/versions", httpGetProviderVersions) + + // GET /:namespace/:name/:provider/:version/download responds with a 204 and X-Terraform-Get header pointing to the download path + r.Get("/{namespace}/{provider}/{version}/download/{os}/{arch}", httpGetProviderDownloadURL) + + // GET /download/ provides an http fileserver for downloading modules as gzipped tarballs + r.Get("/download/{namespace}/{provider}/{system}/{version}/*", httpGetModule) + r.Get("/download/{namespace}/{provider}/{version}/*", httpGetProvider) + + r.Get(config.ProviderBasePath+"/{namespace}/{provider}/{version}/signatures/*", httpGetSignatures) + + // MODULES + r.Get(config.ModuleBasePath+"/{namespace}/{name}/{provider}/versions", httpGetModuleVersions) + + r.Get(config.ModuleBasePath+"/{namespace}/{name}/{provider}/{version}/download", httpGetModuleDownloadURL) + + // FILES + + // r.Get("/files/{file}", httpGetFile) + r.Handle("/files/*", http.FileServer(http.FS(C.S3fsys))) + + err := chi.Walk(r, func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error { + fmt.Printf("[%s]: '%s' has %d middlewares\n", method, route, len(middlewares)) + return nil + }) + if err != nil { + fmt.Println(err) + } + fmt.Println(" > starting TLS server") + + server := &http.Server{ + Addr: ":" + C.Port, + Handler: r, + ReadHeaderTimeout: 3 * time.Second, + } + + err = server.ListenAndServeTLS(C.ServerCert, C.ServerKey) + // err = http.ListenAndServeTLS(":"+C.Port, C.ServerCert, C.ServerKey, r) + if err != nil { + fmt.Println(err) + } + return nil +} + +// httpGetServiceDiscovery is a http handler for returning the +// base path for the modules API provided by this registry +func httpGetServiceDiscovery(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpGetServiceDiscovery") + s := config.ServiceDiscoveryResp{ProvidersV1: config.ProviderBasePath, ModulesV1: config.ModuleBasePath} + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(s) + if err != nil { + fmt.Println(err) + } +} + +func httpIAmAlive(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpIAmAlive") + w.Header().Set("Content-Type", "text") + _, err := w.Write([]byte("internal terraform provier registry is alive")) + if err != nil { + fmt.Println(err) + } +} + +func httpS3Status(w http.ResponseWriter, r *http.Request) { + fmt.Println(" > inside httpS3Status") + statusFile := "status/donotremove.txt" + content, err := fs.ReadFile(C.S3fsys, statusFile) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + w.Header().Set("Content-Type", "text") + _, err = w.Write(content) + if err != nil { + fmt.Println(err) + } +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..87bc795 --- /dev/null +++ b/shell.nix @@ -0,0 +1,19 @@ +# Copyright 2024 SAP SE +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs ? import { } }: + +with pkgs; + +mkShell { + nativeBuildInputs = [ + addlicense + go-licence-detector + go_1_23 + golangci-lint + gotools # goimports + + # keep this line if you use bash + bashInteractive + ]; +}