Merge pull request #2209 from keboola/PSGO-768-proxy-public-wake-up-t…

…est-stability

fix: Proxy wakeup stability of tests

.github/workflows/test-unit.yml

@@ -137,6 +137,7 @@ jobs:
             TEST_KBC_PROJECTS_LOCK_PASSWORD="$TEST_KBC_PROJECTS_LOCK_PASSWORD" \
             TEST_KBC_PROJECTS_FILE="$TEST_KBC_PROJECTS_FILE" \
             TEST_EXCEPT="$TEST_EXCEPT" \
+            TEST_KBC_TMP_DIR="${{ runner.temp }}" \
           make tests-unit
         env:
           TEST_KBC_PROJECTS_LOCK_HOST: ${{ vars.TEST_KBC_PROJECTS_LOCK_HOST }}

docker-compose.yml

@@ -28,6 +28,7 @@ services:
         hard: 50000
     environment:
       # For all
+      - TEST_KBC_TMP_DIR=/tmp
       - TEST_KBC_PROJECTS_FILE=/code/projects.json
       - TEST_KBC_PROJECTS_LOCK_HOST
       - TEST_KBC_PROJECTS_LOCK_PASSWORD

docs/development.md

@@ -20,6 +20,7 @@ Call `make fix` or `go mod vendor` to fix the `vendor` packages. Also when rebas
 
 Create `.env` file with definition of testing projects for example:
 ```
+TEST_KBC_TMP_DIR=/tmp
 TEST_KBC_PROJECTS_FILE=~/keboola-as-code/projects.json
 ```
 

internal/pkg/service/appsproxy/dependencies/mocked.go

@@ -54,7 +54,7 @@ func NewMockedServiceScope(tb testing.TB, ctx context.Context, cfg config.Config
 
 	var dnsServer *dnsmock.Server
 	if cfg.DNSServer == "" {
-		dnsServer = dnsmock.New()
+		dnsServer = dnsmock.New(commonMock.MockedDNSPort())
 		require.NoError(tb, dnsServer.Start())
 		tb.Cleanup(func() {
 			_ = dnsServer.Shutdown()

internal/pkg/service/appsproxy/proxy/proxy_test.go

@@ -14,6 +14,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"path"
 	"strings"
 	"sync"
 	"testing"
@@ -49,6 +50,7 @@ import (
 	"github.com/keboola/keboola-as-code/internal/pkg/service/common/ptr"
 	"github.com/keboola/keboola-as-code/internal/pkg/telemetry"
 	"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/server"
 )
 
 type testCase struct {
@@ -1275,7 +1277,7 @@ func TestAppProxyRouter(t *testing.T) {
 			expectedWakeUps: map[string]int{},
 		},
 		{
-			name: "websocket connection check",
+			name: "websocket-connection-check",
 			run: func(t *testing.T, client *http.Client, m []*mockoidc.MockOIDC, appServer *testutil.AppServer, service *testutil.DataAppsAPI, dnsServer *dnsmock.Server) {
 				ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
 				defer cancel()
@@ -2375,26 +2377,27 @@ func TestAppProxyRouter(t *testing.T) {
 		privateAppTestCaseFactory(http.MethodDelete),
 	)
 
+	tmpDir := path.Join(os.Getenv("TEST_KBC_TMP_DIR"), "TestAppsProxyRouter") // nolint:forbidigo
+	pm, _ := server.NewPortManager(t, tmpDir, "appsproxy")
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-
 			ctx := context.Background()
 
 			// Create testing apps API
-			appsAPI := testutil.StartDataAppsAPI(t)
+			appsAPI := testutil.StartDataAppsAPI(t, pm)
 			t.Cleanup(func() {
 				appsAPI.Close()
 			})
 
 			// Create testing app upstream
-			appServer := testutil.StartAppServer(t)
+			appServer := testutil.StartAppServer(t, pm)
 			t.Cleanup(func() {
 				appServer.Close()
 			})
 
 			// Create dependencies
-			d, mocked := createDependencies(t, ctx, appsAPI.URL)
+			d, mocked := createDependencies(t, ctx, appsAPI.URL, pm)
 
 			// Test generated spans
 			if tc.expectedSpans != nil {
@@ -2405,7 +2408,7 @@ func TestAppProxyRouter(t *testing.T) {
 			dnsServer := mocked.TestDNSServer()
 
 			// Create test OIDC providers
-			providers := testAuthProviders(t)
+			providers := testAuthProviders(t, pm)
 
 			// Register data apps
 			appURL := testutil.AddAppDNSRecord(t, appServer, dnsServer)
@@ -2415,8 +2418,17 @@ func TestAppProxyRouter(t *testing.T) {
 			handler := createProxyHandler(ctx, d)
 
 			// Create a test server for the proxy handler
-			proxySrv := httptest.NewUnstartedServer(handler)
-			proxySrv.Config.ErrorLog = log.NewStdErrorLogger(d.Logger())
+			port := pm.GetFreePort()
+			l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+			for err != nil {
+				port = pm.GetFreePort()
+				l, err = net.Listen("tcp", fmt.Sprintf("[::1]:%d", port))
+			}
+
+			proxySrv := &httptest.Server{
+				Listener: l,
+				Config:   &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second, ErrorLog: log.NewStdErrorLogger(d.Logger())},
+			}
 			proxySrv.StartTLS()
 			t.Cleanup(func() {
 				proxySrv.Close()
@@ -2439,15 +2451,15 @@ func TestAppProxyRouter(t *testing.T) {
 	}
 }
 
-func testAuthProviders(t *testing.T) []*mockoidc.MockOIDC {
+func testAuthProviders(t *testing.T, pm server.PortManager) []*mockoidc.MockOIDC {
 	t.Helper()
 
-	oidc0 := testutil.StartOIDCProviderServer(t)
+	oidc0 := testutil.StartOIDCProviderServer(t, pm)
 	t.Cleanup(func() {
 		require.NoError(t, oidc0.Shutdown())
 	})
 
-	oidc1 := testutil.StartOIDCProviderServer(t)
+	oidc1 := testutil.StartOIDCProviderServer(t, pm)
 	t.Cleanup(func() {
 		require.NoError(t, oidc1.Shutdown())
 	})
@@ -2753,7 +2765,7 @@ func testDataApps(upstream *url.URL, m []*mockoidc.MockOIDC) []api.AppConfig {
 	}
 }
 
-func createDependencies(t *testing.T, ctx context.Context, sandboxesAPIURL string) (proxyDependencies.ServiceScope, proxyDependencies.Mocked) {
+func createDependencies(t *testing.T, ctx context.Context, sandboxesAPIURL string, pm server.PortManager) (proxyDependencies.ServiceScope, proxyDependencies.Mocked) {
 	t.Helper()
 
 	secret := make([]byte, 32)
@@ -2769,8 +2781,8 @@ func createDependencies(t *testing.T, ctx context.Context, sandboxesAPIURL strin
 	cfg.CookieSecretSalt = string(secret)
 	cfg.CsrfTokenSalt = string(csrfSecret)
 	cfg.SandboxesAPI.URL = sandboxesAPIURL
-
-	return proxyDependencies.NewMockedServiceScope(t, ctx, cfg, dependencies.WithRealHTTPClient())
+	port := pm.GetFreePort()
+	return proxyDependencies.NewMockedServiceScope(t, ctx, cfg, dependencies.WithRealHTTPClient(), dependencies.WithMockedDNSPort(port))
 }
 
 func createProxyHandler(ctx context.Context, d proxyDependencies.ServiceScope) http.Handler {

internal/pkg/service/appsproxy/proxy/server_test.go

@@ -23,19 +23,33 @@ import (
 	"github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies"
 	"github.com/keboola/keboola-as-code/internal/pkg/service/common/ptr"
 	"github.com/keboola/keboola-as-code/internal/pkg/telemetry"
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/server"
 )
 
+type portManager struct{}
+
+func newZeroPortManager() server.PortManager {
+	return &portManager{}
+}
+
+func (p portManager) GeneratePorts() {}
+
+func (p portManager) GetFreePort() int {
+	return 0
+}
+
 func TestAppProxyHandler(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
 
 	// Start app
-	appServer := testutil.StartAppServer(t)
+	pm := newZeroPortManager()
+	appServer := testutil.StartAppServer(t, pm)
 	defer appServer.Close()
 
 	// Start api
-	appsAPI := testutil.StartDataAppsAPI(t)
+	appsAPI := testutil.StartDataAppsAPI(t, pm)
 	defer appsAPI.Close()
 
 	// Configure proxy

internal/pkg/service/appsproxy/proxy/testutil/api.go

@@ -3,17 +3,20 @@ package testutil
 import (
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/mitchellh/hashstructure/v2"
 	"github.com/stretchr/testify/require"
 
 	"github.com/keboola/keboola-as-code/internal/pkg/encoding/json"
 	"github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/dataapps/api"
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/server"
 )
 
 type DataAppsAPI struct {
@@ -23,7 +26,7 @@ type DataAppsAPI struct {
 	WakeUps       map[string]int
 }
 
-func StartDataAppsAPI(t *testing.T) *DataAppsAPI {
+func StartDataAppsAPI(t *testing.T, pm server.PortManager) *DataAppsAPI {
 	t.Helper()
 
 	service := &DataAppsAPI{
@@ -79,8 +82,17 @@ func StartDataAppsAPI(t *testing.T) *DataAppsAPI {
 		}
 	})
 
-	ts := httptest.NewUnstartedServer(mux)
-	ts.EnableHTTP2 = true
+	port := pm.GetFreePort()
+	l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+	for err != nil {
+		port = pm.GetFreePort()
+		l, err = net.Listen("tcp", fmt.Sprintf("[::1]:%d", port))
+	}
+	ts := &httptest.Server{
+		Listener:    l,
+		Config:      &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second},
+		EnableHTTP2: true,
+	}
 	ts.Start()
 
 	service.Server = ts

internal/pkg/service/appsproxy/proxy/testutil/app.go

@@ -3,6 +3,7 @@ package testutil
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"sync"
@@ -13,14 +14,16 @@ import (
 	"github.com/coder/websocket/wsjson"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/server"
 )
 
 type AppServer struct {
 	*httptest.Server
 	Requests *[]*http.Request
 }
 
-func StartAppServer(t *testing.T) *AppServer {
+func StartAppServer(t *testing.T, pm server.PortManager) *AppServer {
 	t.Helper()
 
 	lock := &sync.Mutex{}
@@ -53,10 +56,12 @@ func StartAppServer(t *testing.T) *AppServer {
 		for {
 			select {
 			case <-ctx.Done():
-				require.NoError(t, c.Close(websocket.StatusNormalClosure, "Connection closed"))
+				// Ignore error, the websocket can be closed at this point
+				c.Close(websocket.StatusNormalClosure, "Connection closed") //nolint:errcheck
 				return
 			case <-ticker.C:
-				require.NoError(t, wsjson.Write(ctx, c, "Hello from websocket"))
+				// Ignore error, the websocket can be closed at this point
+				wsjson.Write(ctx, c, "Hello from websocket") //nolint:errcheck
 			}
 		}
 	})
@@ -68,8 +73,18 @@ func StartAppServer(t *testing.T) *AppServer {
 		_, _ = fmt.Fprint(w, "Hello, client")
 	})
 
-	ts := httptest.NewUnstartedServer(mux)
-	ts.EnableHTTP2 = true
+	port := pm.GetFreePort()
+	l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+	for err != nil {
+		port = pm.GetFreePort()
+		l, err = net.Listen("tcp", fmt.Sprintf("[::1]:%d", port))
+	}
+
+	ts := &httptest.Server{
+		Listener:    l,
+		Config:      &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second},
+		EnableHTTP2: true,
+	}
 	ts.Start()
 
 	return &AppServer{ts, &requests}

internal/pkg/service/appsproxy/proxy/testutil/dns.go

@@ -9,12 +9,13 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/keboola/keboola-as-code/internal/pkg/service/appsproxy/proxy/transport/dns/dnsmock"
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
 )
 
-func StartDNSServer(t *testing.T) *dnsmock.Server {
+func StartDNSServer(t *testing.T, port int) *dnsmock.Server {
 	t.Helper()
 
-	server := dnsmock.New()
+	server := dnsmock.New(port)
 	err := server.Start()
 	require.NoError(t, err)
 
@@ -31,7 +32,11 @@ func AddAppDNSRecord(t *testing.T, appServer *AppServer, dnsServer *dnsmock.Serv
 	require.NoError(t, err)
 
 	appHost := "app.local"
-	dnsServer.AddARecord(dns.Fqdn(appHost), net.ParseIP(ip))
+	var derr *dnsmock.DNSRecordError
+	err = dnsServer.AddAOrAAAARecord(dns.Fqdn(appHost), net.ParseIP(ip))
+	if err != nil && errors.As(err, &derr) {
+		return nil
+	}
 
 	return &url.URL{
 		Scheme: tsURL.Scheme,

internal/pkg/service/appsproxy/proxy/testutil/oidc.go

@@ -1,17 +1,31 @@
 package testutil
 
 import (
+	"fmt"
+	"net"
 	"testing"
 
 	"github.com/oauth2-proxy/mockoidc"
 	"github.com/stretchr/testify/require"
+
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/server"
 )
 
-func StartOIDCProviderServer(t *testing.T) *mockoidc.MockOIDC {
+func StartOIDCProviderServer(t *testing.T, pm server.PortManager) *mockoidc.MockOIDC {
 	t.Helper()
 
-	m, err := mockoidc.Run()
-	require.NoError(t, err)
+	m, err := mockoidc.NewServer(nil)
+	if err != nil {
+		panic("unable to open mockoidc server" + err.Error())
+	}
+
+	port := pm.GetFreePort()
+	ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+	for err != nil {
+		port = pm.GetFreePort()
+		ln, err = net.Listen("tcp", fmt.Sprintf("[::1]:%d", port))
+	}
 
+	require.NoError(t, m.Start(ln, nil))
 	return m
 }