[Not for merge] Testing CI fixes

dev-tools/mage/common.go

@@ -32,7 +32,7 @@
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -219,10 +219,19 @@
 // HaveDockerCompose returns an error if docker-compose is not found on the
 // PATH.
 func HaveDockerCompose() error {
-	_, err := exec.LookPath("docker-compose")
+
+	path, err := exec.LookPath("docker")
+	if err != nil {
+		return fmt.Errorf("docker is not available")
+	} else {
+		fmt.Printf("hi fae, docker is at %v\n", path)
+	}
+
+	path, err = exec.LookPath("docker-compose")
 	if err != nil {
 		return fmt.Errorf("docker-compose is not available")
 	}
+	fmt.Printf("hi fae, docker compose is %v\n", path)
 	return nil
 }
 
@@ -283,7 +292,7 @@
 func DownloadFile(url, destinationDir string) (string, error) {
 	log.Println("Downloading", url)
 
 	resp, err := http.Get(url)
 	if err != nil {
 		return "", fmt.Errorf("http get failed: %w", err)
 	}
@@ -338,7 +347,7 @@
 		}
 		defer innerFile.Close()
 
 		path := filepath.Join(destinationDir, f.Name)
 		if !strings.HasPrefix(path, destinationDir) {
 			return fmt.Errorf("illegal file path in zip: %v", f.Name)
 		}
@@ -357,7 +366,7 @@
 		}
 		defer out.Close()
 
 		if _, err = io.Copy(out, innerFile); err != nil {
 			return err
 		}
 
@@ -390,7 +399,7 @@
 	tw := tar.NewWriter(zr)
 
 	// walk through every file in the folder
 	filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error {
 		if errFn != nil {
 			return fmt.Errorf("error traversing the file system: %w", errFn)
 		}
@@ -477,13 +486,13 @@
 	for {
 		header, err := tarReader.Next()
 		if err != nil {
 			if err == io.EOF {
 				break
 			}
 			return err
 		}
 
 		path := filepath.Join(destinationDir, header.Name)
 		if !strings.HasPrefix(path, destinationDir) {
 			return fmt.Errorf("illegal file path in tar: %v", header.Name)
 		}
@@ -499,7 +508,7 @@
 				return err
 			}
 
 			if _, err = io.Copy(writer, tarReader); err != nil {
 				return err
 			}
 
@@ -575,7 +584,7 @@
 // based on GOMAXPROCS. The provided ctx is passed to the functions (if they
 // accept it as a param).
 func ParallelCtx(ctx context.Context, fns ...interface{}) {
 	var fnWrappers []func(context.Context) error
 	for _, f := range fns {
 		fnWrapper := funcTypeWrap(f)
 		if fnWrapper == nil {
@@ -800,7 +809,7 @@
 
 	var files []string
 	for _, s := range sources {
 		filepath.Walk(s, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				if os.IsNotExist(err) {
 					return nil

dev-tools/mage/integtest.go

@@ -211,6 +211,7 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio
 
 // NewDockerIntegrationRunner returns an integration runner configured only for docker.
 func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunner, error) {
+	fmt.Printf("hi fae, NewDockerIntegrationRunner\n")
 	cwd, err := os.Getwd()
 	if err != nil {
 		return nil, err
@@ -223,9 +224,11 @@ func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunne
 }
 
 func initRunner(tester IntegrationTester, dir string, passInEnv map[string]string, passThroughEnvVars ...string) (*IntegrationRunner, error) {
+	fmt.Printf("hi fae, initRunner\n")
 	var runnerSteps IntegrationTestSteps
 	requirements := tester.StepRequirements()
 	if requirements != nil {
+		fmt.Printf("hi fae, requirements: %v\n", requirements.Name())
 		runnerSteps = append(runnerSteps, requirements...)
 	}
 

dev-tools/mage/integtest_docker.go

@@ -71,12 +71,15 @@ func (d *DockerIntegrationTester) Use(dir string) (bool, error) {
 
 // HasRequirements ensures that the required docker and docker-compose are installed.
 func (d *DockerIntegrationTester) HasRequirements() error {
+	fmt.Printf("hi fae, in DockerIntegrationTester.HasRequirements\n")
 	if err := HaveDocker(); err != nil {
 		return err
 	}
 	if err := HaveDockerCompose(); err != nil {
 		return err
 	}
+	fmt.Printf("hi fae, everything passed\n")
+	//os.Exit(1)
 	return nil
 }
 
@@ -88,6 +91,7 @@ func (d *DockerIntegrationTester) StepRequirements() IntegrationTestSteps {
 // Test performs the tests with docker-compose. The compose file must define a "beat" container,
 // containing the beats development environment. The tests are executed from within this container.
 func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[string]string) error {
+	fmt.Printf("hi fae, DockerIntegrationTester.Test\n")
 	var err error
 	buildContainersOnce.Do(func() { err = BuildIntegTestContainers() })
 	if err != nil {

libbeat/tests/system/beat/beat.py

@@ -215,9 +215,12 @@ def compose_up_with_retries(cls):
         retries = 3
         for i in range(retries):
             try:
+                print("hi fae, trying compose_up")
                 cls.compose_up()
+                print("hi fae, compose_up succeeded")
                 return
             except Exception as ex:
+                print("hi fae, compose_up failed")
                 if i + 1 >= retries:
                     raise ex
                 print(f"Compose up failed, retrying: {ex}")

libbeat/tests/system/beat/compose.py

@@ -49,39 +49,45 @@ def compose_up(cls):
             return
 
         def print_logs(container):
-            print("---- " + container.name_without_project)
+            print("---- " + container.name)
             print(container.logs())
             print("----")
 
         def is_healthy(container):
-            return container.inspect()['State']['Health']['Status'] == 'healthy'
+            return container.state.health.status == 'healthy' #container.inspect()['State']['Health']['Status'] == 'healthy'
 
         project = cls.compose_project()
 
         with disabled_logger('compose.service'):
             project.pull(
                 ignore_pull_failures=True,
-                service_names=cls.COMPOSE_SERVICES)
+                services=cls.COMPOSE_SERVICES)
 
+        # project.up(
+        #     strategy=ConvergenceStrategy.always,
+        #     service_names=cls.COMPOSE_SERVICES,
+        #     timeout=30)
         project.up(
-            strategy=ConvergenceStrategy.always,
-            service_names=cls.COMPOSE_SERVICES,
-            timeout=30)
+            services=cls.COMPOSE_SERVICES,
+            recreate=True,
+            detach=True,
+        )
 
         # Wait for them to be healthy
         start = time.time()
         while True:
-            containers = project.containers(
-                service_names=cls.COMPOSE_SERVICES,
-                stopped=True)
+            # containers = project.containers(
+            #     service_names=cls.COMPOSE_SERVICES,
+            #     stopped=True)
+            containers = project.ps(services=cls.COMPOSE_SERVICES, all=True)
 
             healthy = True
             for container in containers:
-                if not container.is_running:
+                if not container.state.status == 'running':
                     print_logs(container)
                     raise Exception(
                         "Container %s unexpectedly finished on startup" %
-                        container.name_without_project)
+                        container.name)
                 if not is_healthy(container):
                     healthy = False
                     break
@@ -114,19 +120,10 @@ def _setup_advertised_host(cls, project, service):
         """
         host = cls.compose_host(service=service, port=cls.COMPOSE_ADVERTISED_PORT)
 
-        content = "SERVICE_HOST=%s" % host
-        info = tarfile.TarInfo(name="/run/compose_env")
-        info.mode = 0o100644
-        info.size = len(content)
-
-        data = io.BytesIO()
-        tar = tarfile.TarFile(fileobj=data, mode='w')
-        tar.addfile(info, fileobj=io.BytesIO(content.encode("utf-8")))
-        tar.close()
-
-        containers = project.containers(service_names=[service])
+        containers = project.ps(services=[service])
         for container in containers:
-            container.client.put_archive(container=container.id, path="/", data=data.getvalue())
+            container.execute(["sh", "-c", "echo SERVICE_HOST=%s >/run/compose_env" % host])
+            container.execute(["sh", "-c", "chmod 644 /run/compose_env"])
 
     @classmethod
     def compose_down(cls):
@@ -139,36 +136,37 @@ def compose_down(cls):
         if INTEGRATION_TESTS and cls.COMPOSE_SERVICES:
             # Use down on per-module scenarios to release network pools too
             if os.path.basename(os.path.dirname(cls.find_compose_path())) == "module":
-                cls.compose_project().down(remove_image_type=None, include_volumes=True)
+                cls.compose_project().down(volumes=True)
             else:
-                cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
+                cls.compose_project().kill(services=cls.COMPOSE_SERVICES)
 
     @classmethod
     def get_hosts(cls):
         return [cls.compose_host()]
 
     @classmethod
-    def _private_host(cls, info, port):
+    def _private_host(cls, networks, port):
         """
         Return the address of the container, it should be reachable from the
         host if docker is being run natively. To be used when the tests are
         run from another container in the same network. It also works when
         running from the host network if the docker daemon runs natively.
         """
-        networks = list(info['NetworkSettings']['Networks'].values())
+        #networks = list(info['NetworkSettings']['Networks'].values())
+        networks = list(networks.values())
         port = port.split("/")[0]
         for network in networks:
-            ip = network['IPAddress']
+            ip = network.ip_address
             if ip:
                 return "%s:%s" % (ip, port)
 
     @classmethod
-    def _exposed_host(cls, info, port):
+    def _exposed_host(cls, network_settings, port):
         """
         Return the exposed address in the host, can be used when the test is
         run from the host network. Recommended when using docker machines.
         """
-        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
+        hostPort = network_settings.ports[port][0]['HostPort']
         return "localhost:%s" % hostPort
 
     @classmethod
@@ -183,19 +181,20 @@ def compose_host(cls, service=None, port=None):
         if host_env:
             return host_env
 
-        container = cls.compose_project().containers(service_names=[service])[0]
-        info = container.inspect()
-        portsConfig = info['HostConfig']['PortBindings']
+        container = cls.compose_project().ps(services=[service])[0]
+        # info = container.inspect()
+        portsConfig = container.host_config.port_bindings  #info['HostConfig']['PortBindings']
         if len(portsConfig) == 0:
             raise Exception("No exposed ports for service %s" % service)
         if port is None:
             port = list(portsConfig.keys())[0]
 
         # We can use _exposed_host for all platforms when we can use host network
         # in the metricbeat container
+        # networks = list(info['NetworkSettings']['Networks'].values())
         if sys.platform.startswith('linux'):
-            return cls._private_host(info, port)
-        return cls._exposed_host(info, port)
+            return cls._private_host(container.network_settings.networks, port)
+        return cls._exposed_host(container.network_settings, port)
 
     @classmethod
     def compose_project_name(cls):
@@ -210,9 +209,23 @@ def positivehash(x):
     def compose_project(cls):
         env = Environment(os.environ.copy())
         env.update(cls.COMPOSE_ENV)
-        return get_project(cls.find_compose_path(),
-                           project_name=cls.compose_project_name(),
-                           environment=env)
+
+        compose_files = [os.path.join(cls.find_compose_path(), "docker-compose.yml")]
+
+        print("hi fae, in compose_project")
+        import shutil
+        print("hi fae, docker: %s" % shutil.which("docker"))
+        print("hi fae, docker-compose: %s" % shutil.which("docker-compose"))
+        print("hi fae, isfile(/bin/docker):", os.path.isfile("/bin/docker"))
+
+        from python_on_whales import DockerClient
+        docker = DockerClient(
+            compose_project_name=cls.compose_project_name().lower(),
+            compose_files=compose_files)
+        return docker.compose
+        # return get_project(cls.find_compose_path(),
+        #                    project_name=cls.compose_project_name(),
+        #                    environment=env)
 
     @classmethod
     def find_compose_path(cls):
@@ -226,15 +239,15 @@ def find_compose_path(cls):
 
     @classmethod
     def get_service_log(cls, service):
-        container = cls.compose_project().containers(service_names=[service])[0]
+        container = cls.compose_project().ps(services=[service])[0]
         return container.logs()
 
     @classmethod
     def service_log_contains(cls, service, msg):
         log = cls.get_service_log(service)
         counter = 0
         for line in log.splitlines():
-            if line.find(msg.encode("utf-8")) >= 0:
+            if line.find(msg) >= 0:
                 counter += 1
         return counter > 0
 

libbeat/tests/system/requirements.txt

@@ -1,3 +1,4 @@
+python-on-whales==0.70.1
 async-timeout==4.0.3
 attrs==19.3.0
 autopep8==1.5.4

metricbeat/Dockerfile

@@ -3,6 +3,7 @@ FROM golang:1.21.8
 RUN \
     apt update \
       && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \
+         docker-compose \
          netcat-openbsd \
          python3 \
          python3-dev \

metricbeat/magefile.go

@@ -226,14 +226,20 @@ func GoIntegTest(ctx context.Context) error {
 // Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern.
 // Use any other PYTEST_* environment variable to influence the behavior of pytest.
 func PythonIntegTest(ctx context.Context) error {
+	fmt.Printf("hi fae, starting PythonIntegTest\n")
 	if !devtools.IsInIntegTestEnv() {
+		fmt.Printf("hi fae, !IsInIntegTestEnv\n")
 		mg.SerialDeps(Fields, Dashboards)
+	} else {
+		fmt.Printf("hi fae, IsInIntegTestEnv\n")
 	}
 
 	passThroughEnvVars := append(
 		[]string{"ELASTICSEARCH_VERSION", "KIBANA_VERSION", "BEAT_VERSION"},
 		devtools.ListMatchingEnvVars("PYTEST_")...,
 	)
+	fmt.Printf("hi fae, env vars: %v\n", passThroughEnvVars)
+	//os.Exit(1)
 	runner, err := devtools.NewDockerIntegrationRunner(passThroughEnvVars...)
 	if err != nil {
 		return err