Merge pull request 'Initial testing and improvements of code' (#1) fr…

…om feature/testing into master

Reviewed-on: https://git.meow.tf/tyler/armbian-router/pulls/1

.drone.yml

@@ -3,6 +3,18 @@ name: default
 type: docker
 
 steps:
+  - name: test
+    image: golang:alpine
+    volumes:
+      - name: build
+        path: /build
+    commands:
+      - go mod download
+      - go install github.com/onsi/ginkgo/v2/ginkgo
+      - ginkgo --randomize-all --p --cover --coverprofile=cover.out .
+      - go tool cover -func=cover.out
+    environment:
+      CGO_ENABLED: '0'
   - name: build
     image: tystuyfzand/goc:latest
     volumes:
@@ -16,7 +28,7 @@ steps:
     environment:
       GOOS: linux,windows,darwin
       GOARCH: 386,amd64,arm,arm64
-    depends_on: [ clone ]
+    depends_on: [ test ]
   - name: release
     image: plugins/gitea-release
     volumes:
@@ -48,7 +60,7 @@ steps:
         from_secret: docker_password
       repo: registry.meow.tf/tyler/armbian-router
       registry: registry.meow.tf
-    depends_on: [ clone ]
+    depends_on: [ test ]
     when:
       event: tag
 volumes:

.gitignore

@@ -2,4 +2,5 @@
 userdata.csv
 dlrouter-apt.yaml
 *.yaml
-!dlrouter.yaml
+!dlrouter.yaml
+*.exe

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2022 Tyler Stuyfzand <[email protected]>
+Copyright (c) 2022 Tyler Stuyfzand <[email protected]>, Armbian Project
 
 Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
 

README.md

@@ -0,0 +1,118 @@
+Armbian Redirector
+==================
+
+This repository contains a redirect service for Armbian downloads, apt, etc.
+
+It uses multiple current technologies and best practices, including:
+
+- Go 1.17/1.18
+- GeoIP + Distance routing
+- Server weighting, pooling (top x servers are served instead of a single one)
+- Health checks (HTTP, TLS)
+
+Code Quality
+------------
+
+The code quality isn't the greatest/top tier. All code lives in the "main" package and should be moved at some point.
+
+Regardless, it is meant to be simple and easy to understand.
+
+Configuration
+-------------
+
+### Modes
+
+#### Redirect
+
+Standard redirect functionality
+
+#### Download Mapping
+
+Uses the `dl_map` configuration variable to enable mapping of paths to new paths.
+
+Think symlinks, but in a generated file.
+
+### Mirrors
+Mirror targets with trailing slash are placed in the yaml configuration file.
+
+### Example YAML
+```yaml
+# GeoIP Database Path
+geodb: GeoLite2-City.mmdb
+
+# Comment out to disable
+dl_map: userdata.csv
+
+# LRU Cache Size (in items)
+cacheSize: 1024
+
+# Server definition
+# Weights are just like nginx, where if it's > 1 it'll be chosen x out of x + total times
+# By default, the top 3 servers are used for choosing the best.
+# server = full url or host+path
+# weight = int
+# optional: latitude, longitude (float)
+servers:
+  - server: armbian.12z.eu/apt/
+  - server: armbian.chi.auroradev.org/apt/
+    weight: 15
+    latitude: 41.8879
+    longitude: -88.1995
+````
+
+## API
+
+`/status`
+
+Meant for a simple health check (nginx/etc can 502 or similar if down)
+
+`/reload`
+
+Flushes cache and reloads configuration and mapping. Requires reloadToken to be set in the configuration, and a matching token provided in `Authorization: Bearer TOKEN`
+
+`/mirrors`
+
+Shows all mirrors in the legacy (by region) format
+
+`/mirrors.json`
+
+Shows all mirrors in the new JSON format. Example:
+
+```json
+[
+  {
+    "available":true,
+    "host":"imola.armbian.com",
+    "path":"/apt/",
+    "latitude":46.0503,
+    "longitude":14.5046,
+    "weight":10,
+    "continent":"EU",
+    "lastChange":"2022-08-12T06:52:35.029565986Z"
+  }
+]
+```
+
+`/mirrors/{server}.svg`
+
+Magic SVG path to show badges based on server status, for use in dynamic mirror lists.
+
+`/dl_map`
+
+Shows json-encoded download mappings
+
+`/geoip`
+
+Shows GeoIP information for the requester
+
+`/region/REGIONCODE/PATH`
+
+Using this magic path will redirect to the desired region:
+
+* NA - North America
+* EU - Europe
+* AS - Asia
+
+`/metrics`
+
+Prometheus metrics endpoint. Metrics aren't considered private, thus are exposed to the public.

armbianmirror_suite_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestArmbianMirror(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ArmbianMirror Suite")
+}

check.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"net"
+	"net/http"
+	"net/url"
+	"runtime"
+	"time"
+)
+
+var (
+	ErrHttpsRedirect = errors.New("unexpected forced https redirect")
+	ErrCertExpired   = errors.New("certificate is expired")
+)
+
+// checkHttp checks a URL for validity, and checks redirects
+func checkHttp(server *Server, logFields log.Fields) (bool, error) {
+	u := &url.URL{
+		Scheme: "http",
+		Host:   server.Host,
+		Path:   server.Path,
+	}
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+
+	req.Header.Set("User-Agent", "ArmbianRouter/1.0 (Go "+runtime.Version()+")")
+
+	if err != nil {
+		return false, err
+	}
+
+	res, err := checkClient.Do(req)
+
+	if err != nil {
+		return false, err
+	}
+
+	logFields["responseCode"] = res.StatusCode
+
+	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusNotFound {
+		if res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound {
+			location := res.Header.Get("Location")
+
+			logFields["url"] = location
+
+			// Check that we don't redirect to https from a http url
+			if u.Scheme == "http" {
+				res, err := checkRedirect(location)
+
+				if !res || err != nil {
+					return res, err
+				}
+			}
+		}
+
+		return true, nil
+	}
+
+	logFields["cause"] = fmt.Sprintf("Unexpected http status %d", res.StatusCode)
+
+	return false, nil
+}
+
+// checkRedirect parses a location header response and checks the scheme
+func checkRedirect(locationHeader string) (bool, error) {
+	newUrl, err := url.Parse(locationHeader)
+
+	if err != nil {
+		return false, err
+	}
+
+	if newUrl.Scheme == "https" {
+		return false, ErrHttpsRedirect
+	}
+
+	return true, nil
+}
+
+// checkTLS checks tls certificates from a host, ensures they're valid, and not expired.
+func checkTLS(server *Server, logFields log.Fields) (bool, error) {
+	host, port, err := net.SplitHostPort(server.Host)
+
+	if port == "" {
+		port = "443"
+	}
+
+	conn, err := tls.Dial("tcp", host+":"+port, checkTLSConfig)
+
+	if err != nil {
+		return false, err
+	}
+
+	defer conn.Close()
+
+	err = conn.VerifyHostname(server.Host)
+
+	if err != nil {
+		return false, err
+	}
+
+	now := time.Now()
+
+	state := conn.ConnectionState()
+
+	opts := x509.VerifyOptions{
+		CurrentTime: time.Now(),
+	}
+
+	for _, cert := range state.PeerCertificates {
+		if _, err := cert.Verify(opts); err != nil {
+			logFields["peerCert"] = cert.Subject.String()
+			return false, err
+		}
+		if now.Before(cert.NotBefore) || now.After(cert.NotAfter) {
+			return false, err
+		}
+	}
+
+	for _, chain := range state.VerifiedChains {
+		for _, cert := range chain {
+			if now.Before(cert.NotBefore) || now.After(cert.NotAfter) {
+				logFields["cert"] = cert.Subject.String()
+				return false, ErrCertExpired
+			}
+		}
+	}
+
+	return true, nil
+}