Merge pull request #1 from armbian/feature/refactor

Refactoring and half rewrite

.drone.yml

@@ -10,7 +10,7 @@ steps:
         path: /build
     commands:
       - go mod download
-      - go install github.com/onsi/ginkgo/v2/ginkgo
+      - go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
       - ginkgo --randomize-all --p --cover --coverprofile=cover.out .
       - go tool cover -func=cover.out
     environment:

.github/workflows/main.yml

@@ -0,0 +1,83 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CICD
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches:
+      - master
+      - staging
+      - develop
+  pull_request:
+    branches:
+      - master
+      - staging
+      - develop
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # The "build" workflow
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      # Setup Go
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.18' # The Go version to download (if necessary) and use.
+
+      # Install all the dependencies
+      - name: Install dependencies
+        run: |
+          go version
+          go install -mod=mod golang.org/x/lint/golint
+          go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      # Run vet & lint on the code
+      - name: Run vet & lint
+        run: |
+          go vet .
+          golint .
+
+      # Run testing on the code
+      - name: Run testing
+        run: |
+          ginkgo --randomize-all --p --cover --coverprofile=cover.out .
+          go tool cover -func=cover.out
+
+      # Install build tool
+      - name: Install build tool
+        run: go install github.com/tystuyfzand/goc@latest
+
+      # Run build of the application
+      - name: Run build
+        run: |
+          mkdir build/
+          goc -o build/dlrouter cmd/main.go
+        env:
+          GOOS: linux,windows,darwin,openbsd,freebsd
+          GOARCH: 386,amd64,arm,arm64
+
+      # Upload artifacts
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: binaries
+          path: build/
+
+  # The "deploy" workflow
+  deploy:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    needs: [build] # Only run this workflow when "build" workflow succeeds
+    if: ${{ github.ref == 'refs/heads/master' && github.event_name == 'push' }} # Only run this workflow if it is master branch on push event
+    steps:
+      - uses: actions/checkout@v2

.gitignore

@@ -3,4 +3,6 @@ userdata.csv
 dlrouter-apt.yaml
 *.yaml
 !dlrouter.yaml
-*.exe
+*.exe
+dlrouter
+cover.out

README.md

@@ -5,17 +5,37 @@ This repository contains a redirect service for Armbian downloads, apt, etc.
 
 It uses multiple current technologies and best practices, including:
 
-- Go 1.17/1.18
+- Go 1.19
+- Ginkgo v2 and Gomega testing framework
 - GeoIP + Distance routing
 - Server weighting, pooling (top x servers are served instead of a single one)
 - Health checks (HTTP, TLS)
 
 Code Quality
 ------------
 
-The code quality isn't the greatest/top tier. All code lives in the "main" package and should be moved at some point.
+The code quality isn't the greatest/top tier. Work is being done towards cleaning it up and standardizing it, writing tests, etc.
 
-Regardless, it is meant to be simple and easy to understand.
+All contributions are welcome, see the `check_test.go` file for example tests.
+
+Checks
+------
+
+The supported checks are HTTP and TLS.
+
+### HTTP
+
+Verifies server accessibility via HTTP. If the server returns a forced redirect to an `https://` url, it is considered to be https-only.
+
+If the server responds on the `https` url with a forced `http` redirect, it will be marked down due to misconfiguration. Requests should never downgrade.
+
+### TLS
+
+Certificate checking to ensure no servers are used which have invalid/expired certificates. This check is written to use the Mozilla ca certificate list, loaded on start/config load, to verify roots.
+
+OS certificate trusts WERE being used to do this, however some issues with the date validation (which could be user error) caused the move to the ca bundle, which could be considered more usable.
+
+Note: This downloads from github every startup/reload. This should be a reliable process, as long as Mozilla doesn't deprecate their repo. Their HG URL is super slow.
 
 Configuration
 -------------
@@ -52,12 +72,19 @@ cacheSize: 1024
 # server = full url or host+path
 # weight = int
 # optional: latitude, longitude (float)
+# optional: protocols (list/array)
 servers:
   - server: armbian.12z.eu/apt/
   - server: armbian.chi.auroradev.org/apt/
     weight: 15
     latitude: 41.8879
     longitude: -88.1995
+  # Example of a server with additional protocols (rsync)
+  # Useful for defining servers which could be used for rsync sources
+  - server: mirrors.dotsrc.org/armbian-apt/
+    weight: 15
+    protocols:
+      - rsync
 ````
 
 ## API

armbianmirror_suite_test.go

@@ -1,4 +1,4 @@
-package main
+package redirector
 
 import (
 	"testing"

check.go

@@ -1,4 +1,4 @@
-package main
+package redirector
 
 import (
 	"crypto/tls"
@@ -10,18 +10,34 @@ import (
 	"net/http"
 	"net/url"
 	"runtime"
+	"strings"
 	"time"
 )
 
 var (
-	ErrHttpsRedirect = errors.New("unexpected forced https redirect")
-	ErrCertExpired   = errors.New("certificate is expired")
+	// ErrHTTPSRedirect is an error thrown when the webserver returns
+	// an https redirect for an http url.
+	ErrHTTPSRedirect = errors.New("unexpected forced https redirect")
+
+	// ErrHTTPRedirect is an error thrown when the webserver returns
+	// a redirect to a non-https url from an https request.
+	ErrHTTPRedirect = errors.New("unexpected redirect to insecure url")
+
+	// ErrCertExpired is a fatal error thrown when the webserver's
+	// certificate is expired.
+	ErrCertExpired = errors.New("certificate is expired")
 )
 
-// checkHttp checks a URL for validity, and checks redirects
-func checkHttp(server *Server, logFields log.Fields) (bool, error) {
+func (r *Redirector) checkHTTP(scheme string) ServerCheck {
+	return func(server *Server, logFields log.Fields) (bool, error) {
+		return r.checkHTTPScheme(server, scheme, logFields)
+	}
+}
+
+// checkHTTPScheme checks a URL for validity, and checks redirects
+func (r *Redirector) checkHTTPScheme(server *Server, scheme string, logFields log.Fields) (bool, error) {
 	u := &url.URL{
-		Scheme: "http",
+		Scheme: scheme,
 		Host:   server.Host,
 		Path:   server.Path,
 	}
@@ -34,27 +50,34 @@ func checkHttp(server *Server, logFields log.Fields) (bool, error) {
 		return false, err
 	}
 
-	res, err := checkClient.Do(req)
+	res, err := r.config.checkClient.Do(req)
 
 	if err != nil {
 		return false, err
 	}
 
 	logFields["responseCode"] = res.StatusCode
 
-	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusNotFound {
-		if res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound {
+	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusPermanentRedirect || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusNotFound {
+		if res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusPermanentRedirect {
 			location := res.Header.Get("Location")
 
 			logFields["url"] = location
 
-			// Check that we don't redirect to https from a http url
-			if u.Scheme == "http" {
-				res, err := checkRedirect(location)
+			switch u.Scheme {
+			case "http":
+				res, err := r.checkRedirect(u.Scheme, location)
 
 				if !res || err != nil {
-					return res, err
+					// If we don't support http, we remove it from supported protocols
+					server.Protocols = server.Protocols.Remove("http")
+				} else {
+					// Otherwise, we verify https support
+					r.checkProtocol(server, "https")
 				}
+			case "https":
+				// We don't want to allow downgrading, so this is an error.
+				return r.checkRedirect(u.Scheme, location)
 			}
 		}
 
@@ -66,30 +89,63 @@ func checkHttp(server *Server, logFields log.Fields) (bool, error) {
 	return false, nil
 }
 
+func (r *Redirector) checkProtocol(server *Server, scheme string) {
+	res, err := r.checkHTTPScheme(server, scheme, log.Fields{})
+
+	if !res || err != nil {
+		return
+	}
+
+	if !server.Protocols.Contains(scheme) {
+		server.Protocols = server.Protocols.Append(scheme)
+	}
+}
+
 // checkRedirect parses a location header response and checks the scheme
-func checkRedirect(locationHeader string) (bool, error) {
-	newUrl, err := url.Parse(locationHeader)
+func (r *Redirector) checkRedirect(originatingScheme, locationHeader string) (bool, error) {
+	newURL, err := url.Parse(locationHeader)
 
 	if err != nil {
 		return false, err
 	}
 
-	if newUrl.Scheme == "https" {
-		return false, ErrHttpsRedirect
+	if newURL.Scheme == "https" {
+		return false, ErrHTTPSRedirect
+	} else if originatingScheme == "https" && newURL.Scheme == "http" {
+		return false, ErrHTTPRedirect
 	}
 
 	return true, nil
 }
 
 // checkTLS checks tls certificates from a host, ensures they're valid, and not expired.
-func checkTLS(server *Server, logFields log.Fields) (bool, error) {
-	host, port, err := net.SplitHostPort(server.Host)
+func (r *Redirector) checkTLS(server *Server, logFields log.Fields) (bool, error) {
+	var host, port string
+	var err error
+
+	if strings.Contains(server.Host, ":") {
+		host, port, err = net.SplitHostPort(server.Host)
+
+		if err != nil {
+			return false, err
+		}
+	} else {
+		host = server.Host
+	}
+
+	log.WithFields(log.Fields{
+		"server": server.Host,
+		"host":   host,
+		"port":   port,
+	}).Debug("Checking TLS server")
 
 	if port == "" {
 		port = "443"
 	}
 
-	conn, err := tls.Dial("tcp", host+":"+port, checkTLSConfig)
+	conn, err := tls.Dial("tcp", host+":"+port, &tls.Config{
+		RootCAs: r.config.RootCAs,
+	})
 
 	if err != nil {
 		return false, err
@@ -107,18 +163,38 @@ func checkTLS(server *Server, logFields log.Fields) (bool, error) {
 
 	state := conn.ConnectionState()
 
+	peerPool := x509.NewCertPool()
+
+	for _, intermediate := range state.PeerCertificates {
+		if !intermediate.IsCA {
+			continue
+		}
+
+		peerPool.AddCert(intermediate)
+	}
+
 	opts := x509.VerifyOptions{
-		CurrentTime: time.Now(),
+		Roots:         r.config.RootCAs,
+		Intermediates: peerPool,
+		CurrentTime:   time.Now(),
 	}
 
-	for _, cert := range state.PeerCertificates {
-		if _, err := cert.Verify(opts); err != nil {
-			logFields["peerCert"] = cert.Subject.String()
-			return false, err
-		}
-		if now.Before(cert.NotBefore) || now.After(cert.NotAfter) {
-			return false, err
+	// We want only the leaf certificate, as this will verify up the chain for us.
+	cert := state.PeerCertificates[0]
+
+	if _, err := cert.Verify(opts); err != nil {
+		logFields["peerCert"] = cert.Subject.String()
+
+		if authErr, ok := err.(x509.UnknownAuthorityError); ok {
+			logFields["authCert"] = authErr.Cert.Subject.String()
+			logFields["ca"] = authErr.Cert.Issuer
 		}
+		return false, err
+	}
+
+	if now.Before(cert.NotBefore) || now.After(cert.NotAfter) {
+		logFields["peerCert"] = cert.Subject.String()
+		return false, err
 	}
 
 	for _, chain := range state.VerifiedChains {
@@ -130,5 +206,10 @@ func checkTLS(server *Server, logFields log.Fields) (bool, error) {
 		}
 	}
 
+	// If https is valid, append it
+	if !server.Protocols.Contains("https") {
+		server.Protocols = server.Protocols.Append("https")
+	}
+
 	return true, nil
 }