diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e7dff1f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +**/.git** +**/Dockerfile +**/README +**/*.md + +.github +docker* +e2e-test +examples +tools +!tools/setup diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b5c6f36 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,103 @@ +# Share args between stages. +# See https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact. +# They should be consistent with tools/setup/p4sde_env_setup.sh. +ARG SDE=/home/sde +ARG SDE_INSTALL=${SDE}/install +ARG LD_LIBRARY_PATH=${SDE_INSTALL}/lib:${SDE_INSTALL}/lib64:${SDE_INSTALL}/lib/x86_64-linux-gnu/ +ARG LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64 +ARG LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib + +#====================================== +# Stage 1/3: Reuse upstream p4c image. +#====================================== +# FROM p4lang/p4c:latest as p4c +# :latest on 2023-06-23: +FROM p4lang/p4c@sha256:c1cbb66cea83de50b43d7ef78d478dd5e43ce9e1116921d6700cc40bb505e12a as p4c + +#============================================== +# Stage 2/3: Build P4 DPDK target from source. +#============================================== +FROM ubuntu:20.04 as p4-dpdk-target +ARG DEBIAN_FRONTEND=noninteractive +ARG SDE +ARG SDE_INSTALL +ARG LD_LIBRARY_PATH + +# Install core packages needed for later steps. +RUN apt-get update && apt-get install -y \ + python-is-python3 \ + python3 \ + python3-pip \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +# Copy repo files in. +ARG REPO_NAME=p4-dpdk-target +COPY . ${SDE}/${REPO_NAME}/ + +# Install dependent packages. +WORKDIR ${SDE}/${REPO_NAME}/tools/setup +RUN pip3 install distro \ + && apt-get update \ + && python3 install_dep.py \ + && rm -rf /var/lib/apt/lists/* + +# Build P4 DPDK target. +WORKDIR ${SDE}/${REPO_NAME} +RUN ./autogen.sh \ + && ./configure --prefix=${SDE_INSTALL} \ + && make -j \ + && make install + +#=================================== +# Stage 3/3: Construct final image. +#=================================== +FROM ubuntu:20.04 +LABEL maintainer="P4 Developers " +ARG DEBIAN_FRONTEND=noninteractive +ARG SDE +ARG SDE_INSTALL +ARG LD_LIBRARY_PATH + +# Persist essential environment variables. +ENV SDE=${SDE} +ENV SDE_INSTALL=${SDE_INSTALL} +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH} + +# Install some useful tools. +RUN apt-get update && apt-get install -y \ + iproute2 \ + python-is-python3 \ + python3 \ + python3-pip \ + sudo \ + && rm -rf /var/lib/apt/lists/* +RUN pip3 install --upgrade \ + scapy + +# [from p4c] +# Just copy minimal p4c bins & libs for dpdk backend. +# Adapted from https://github.com/sonic-net/DASH/blob/main/dash-pipeline/dockerfiles/Dockerfile.p4c-dpdk. +COPY --from=p4c \ + /usr/local/bin/p4c \ + /usr/local/bin/p4c-dpdk \ + /usr/local/bin/ +COPY --from=p4c \ + /usr/lib/x86_64-linux-gnu/libboost_*so* \ + /usr/lib/x86_64-linux-gnu/libgc.so* \ + /usr/lib/x86_64-linux-gnu/ +COPY --from=p4c \ + /usr/local/share/p4c/ \ + /usr/local/share/p4c/ + +# [from p4-dpdk-target] +# Just copy SDE_INSTALL dir & minimal system libs. +COPY --from=p4-dpdk-target \ + ${SDE_INSTALL} \ + ${SDE_INSTALL} +COPY --from=p4-dpdk-target \ + /usr/lib/x86_64-linux-gnu/libedit.so* \ + /usr/lib/x86_64-linux-gnu/ + +# Set default working directory. +WORKDIR ${SDE} diff --git a/bf_switchd/bf_switchd.c b/bf_switchd/bf_switchd.c index 985bae4..769c3ee 100644 --- a/bf_switchd/bf_switchd.c +++ b/bf_switchd/bf_switchd.c @@ -1565,15 +1565,6 @@ static int bf_switchd_sys_init(void) { return ret; } - /* For performance set defaut log level to error for pipe and pkt modules */ - bf_sys_log_level_set(BF_MOD_PIPE, BF_LOG_DEST_FILE, BF_LOG_ERR); - bf_sys_trace_level_set(BF_MOD_PIPE, BF_LOG_ERR); - -#ifdef BFRT_ENABLED - bf_sys_log_level_set(BF_MOD_BFRT, BF_LOG_DEST_FILE, BF_LOG_ERR); - bf_sys_trace_level_set(BF_MOD_BFRT, BF_LOG_ERR); -#endif - return 0; } diff --git a/e2e-test/README.md b/e2e-test/README.md new file mode 100644 index 0000000..166c6a5 --- /dev/null +++ b/e2e-test/README.md @@ -0,0 +1,129 @@ +# P4 DPDK Target End-to-End Testing + +## Directory structure + +``` +suites/ + Collection of test suites. + + // + Individual test suite. + + (generated) log/ + Log files generated during testing. + (generated) p4c_gen/ + P4 compiler outputs generated from main.p4. + cmds_bfshell.py: + Commands to run with bfshell before testing script. + cmds_shell.sh: + Commands to run with normal shell before testing script. + (generated) conf_bf_switchd.json: + Config file needed by the bf_switchd binary. + main.p4: + The P4 pipeline code. + README.md: + A description of the test. + test.py: + The packet testing script. + +tools/ + Tools used to run the tests. +``` + +## Testing environement + +You can either build P4 DPDK target locally following the README at the root of this repo, or build a Docker image. Below we introduce the Docker approach. + +All commands in this section are expected to be executed from the root of this repo. + +### Docker image building + +```console +./tools/docker_build.sh +``` + +### Convenient Docker scripts + +To run a `bash` in a new container: + +```console +./tools/docker_run_bash.sh +``` + +To run a new `bash` in the same running container: + +```console +./tools/docker_exec_bash.sh +``` + +The last script is useful for providing multiple terminal shells in the same container. + +## Simple automated smoke test + +To run the smoke test in a container, at the **root of this repo**, do: + +```console +./tools/docker_run_smoke_test.sh +``` + +To run it locally, at **this directory**, do: + +```console +./tools/test.py suites/pna/basicfwd +``` + +## Manual testing workflow + +In the previous section, all testing steps are executed with a single script. If for some reason (e.g. debugging) you would like to execute these steps separately, the details are described in this section. + +All commands in this section are expected to be executed from this directory. + +### Compile P4 code + +```console +./tools/compile.py suites/pna/basicfwd +``` + +The following paths will be generated: + +- `suites/pna/basicfwd/p4c_gen/` + +### Run `bf_switchd` + +In terminal 1, do: + +```console +./tools/run_bf_switchd.py suites/pna/basicfwd +``` + +The following paths will be generated: + +- `suites/pna/simple_l2_forwarding/log/bf_switchd/` +- `suites/pna/simple_l2_forwarding/conf_bf_switchd.json` + +### Run `bfshell` + +In terminal 2, do: + +```console +./tools/run_bfshell.py suites/pna/basicfwd +``` + +The following paths will be generated: + +- `suites/pna/simple_l2_forwarding/log/bfshell/` + +### Run testing script + +In terminal 3, do: + +```console +./suites/pna/basicfwd/cmds_shell.sh +./suites/pna/basicfwd/test.py +``` + +### Clean up (optional) + +```console +./tools/clean.py suites/pna/basicfwd +``` diff --git a/e2e-test/suites/.gitignore b/e2e-test/suites/.gitignore new file mode 100644 index 0000000..f832861 --- /dev/null +++ b/e2e-test/suites/.gitignore @@ -0,0 +1,3 @@ +log/ +p4c_gen/ +conf_bf_switchd.json diff --git a/e2e-test/suites/pna/basicfwd/README.md b/e2e-test/suites/pna/basicfwd/README.md new file mode 100644 index 0000000..e63eb46 --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/README.md @@ -0,0 +1,3 @@ +# Basic Forwarding Test + +Receive packets on a port and forward them on the paired port. The mapping is 0 -> 1, 1 -> 0, 2 -> 3, 3 -> 2, etc. diff --git a/e2e-test/suites/pna/basicfwd/cmds_bfshell.py b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py new file mode 100644 index 0000000..4a3e240 --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py @@ -0,0 +1,32 @@ +tdi_python + +# Add ports +for port_id in range(4): + tdi.port.port.add( + DEV_PORT=port_id, + PORT_TYPE="BF_DPDK_TAP", + PORT_DIR="PM_PORT_DIR_DEFAULT", + PORT_IN_ID=port_id, + PORT_OUT_ID =port_id, + PIPE_IN="pipe", + PIPE_OUT="pipe", + MEMPOOL="MEMPOOL0", + PORT_NAME=f"TAP{port_id}", + MTU=1500 + ) + +# Enable TDI program +tdi.main.enable() + +# Add entries +control = tdi.main.pipe.MainControl +table = control.forwarding +table.add_with_forward(input_port=0, output_port=1) +table.add_with_forward(input_port=1, output_port=0) +table.add_with_forward(input_port=2, output_port=3) +table.add_with_forward(input_port=3, output_port=2) + +# TODO: Replace the following hack with something better. +foo = "fini" +bar = "shed" +print(foo + bar) diff --git a/e2e-test/suites/pna/basicfwd/cmds_shell.sh b/e2e-test/suites/pna/basicfwd/cmds_shell.sh new file mode 100755 index 0000000..6f72c7e --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/cmds_shell.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Set up TAP ports +for port in {0..3}; do + sudo ip link set "TAP$port" up +done + +# Disable IPv6 because it can interfere with packet sniffing +sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 +sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 +sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1 diff --git a/e2e-test/suites/pna/basicfwd/main.p4 b/e2e-test/suites/pna/basicfwd/main.p4 new file mode 100644 index 0000000..1b0df3f --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/main.p4 @@ -0,0 +1,83 @@ +#include +#include + +header eth_h { + bit<48> dst_addr; + bit<48> src_addr; + bit<16> ether_type; +} + +struct header_t { + eth_h eth; +} + +struct metadata_t {} + +parser MainParser( + packet_in pkt, + out header_t hdr, + inout metadata_t meta, + in pna_main_parser_input_metadata_t istd +) { + state start { + pkt.extract(hdr.eth); + transition accept; + } +} + +control PreControl( + in header_t hdr, + inout metadata_t meta, + in pna_pre_input_metadata_t istd, + inout pna_pre_output_metadata_t ostd +) { + apply {} +} + +control MainControl( + inout header_t hdr, + inout metadata_t meta, + in pna_main_input_metadata_t istd, + inout pna_main_output_metadata_t ostd +) { + action forward(PortId_t output_port) { + send_to_port(output_port); + } + + action drop() { + drop_packet(); + } + + table forwarding { + key = { + istd.input_port: exact; + } + actions = { + forward; + drop; + } + const default_action = drop(); + } + + apply { + forwarding.apply(); + } +} + +control MainDeparser( + packet_out pkt, + in header_t hdr, + in metadata_t meta, + in pna_main_output_metadata_t ostd +) { + apply { + pkt.emit(hdr); + } +} + +PNA_NIC( + MainParser(), + PreControl(), + MainControl(), + MainDeparser() +) main; diff --git a/e2e-test/suites/pna/basicfwd/test.py b/e2e-test/suites/pna/basicfwd/test.py new file mode 100755 index 0000000..0bd8bf8 --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/test.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import time +import unittest + +from scapy.all import IP, UDP, AsyncSniffer, Ether, raw, sendp + + +class ScapyTestCase(unittest.TestCase): + WAIT_TIME = 1 + + def test_one_packet_received_at_expected_port(self): + sniffer = AsyncSniffer(iface="TAP1", count=8) + sniffer.start() + + pkt = Ether() / IP() / UDP() / "Hello, World!" + sendp(pkt, iface="TAP0") + pkt_sent = pkt + + time.sleep(self.WAIT_TIME) + pkts_recvd = sniffer.stop() + + self.assertEqual(len(pkts_recvd), 1) + pkt_recvd = pkts_recvd[0] + self.assertEqual(raw(pkt_recvd), raw(pkt_sent)) + + def test_no_packet_received_at_other_ports(self): + sniffer = AsyncSniffer(iface=["TAP2", "TAP3"], count=8) + sniffer.start() + + pkt = Ether() / IP() / UDP() / "Hello, World!" + sendp(pkt, iface="TAP0") + + time.sleep(self.WAIT_TIME) + pkts_recvd = sniffer.stop() + + self.assertEqual(len(pkts_recvd), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tools/.gitignore b/e2e-test/tools/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/e2e-test/tools/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/e2e-test/tools/clean.py b/e2e-test/tools/clean.py new file mode 100755 index 0000000..0185ccf --- /dev/null +++ b/e2e-test/tools/clean.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import argparse +import logging +from pathlib import Path + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + clean(args.test_dir) + + +def clean(test_dir): + util.assert_exists(test_dir) + + for path in config.SUITE_PATH_GENERATED: + util.remove_file_or_dir(test_dir / path) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/compile.py b/e2e-test/tools/compile.py new file mode 100755 index 0000000..bee2c13 --- /dev/null +++ b/e2e-test/tools/compile.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import argparse +import logging +from pathlib import Path +from subprocess import run + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + compile(args.test_dir) + + +def compile(test_dir): + util.assert_exists(test_dir) + + # Prepare compiler output dir + output_dir = test_dir / config.SUITE_P4C_GEN + if output_dir.exists(): + logging.info(f"Removing existing compiler output dir {output_dir}") + util.remove_file_or_dir(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Compile P4 source + arch = test_dir.parent.stem + p4_source = test_dir / config.SUITE_MAIN_P4 + cmd = ["p4c-dpdk"] + cmd += ["--arch", arch] + cmd += ["--p4runtime-files", test_dir / config.SUITE_P4INFO] + cmd += ["--bf-rt-schema", test_dir / config.SUITE_BFRT] + cmd += ["--tdi", test_dir / config.SUITE_TDI] + cmd += ["--context", test_dir / config.SUITE_CONTEXT] + cmd += ["-o", test_dir / config.SUITE_SPEC] + cmd += [p4_source] + util.log_cmd(cmd) + run(cmd, check=True) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/config.py b/e2e-test/tools/config.py new file mode 100644 index 0000000..3faa463 --- /dev/null +++ b/e2e-test/tools/config.py @@ -0,0 +1,22 @@ +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent + +# Test suite path conventions + +SUITE_MAIN_P4 = "main.p4" +SUITE_CMDS_BFSHELL = "cmds_bfshell.py" +SUITE_CMDS_SHELL = "cmds_shell.sh" +SUITE_CONF_BF_SWITCHD = "conf_bf_switchd.json" +SUITE_TEST = "test.py" + +SUITE_P4C_GEN = "p4c_gen" +SUITE_P4INFO = SUITE_P4C_GEN + "/p4info.txt" +SUITE_BFRT = SUITE_P4C_GEN + "/bfrt.json" +SUITE_TDI = SUITE_P4C_GEN + "/tdi.json" +SUITE_CONTEXT = SUITE_P4C_GEN + "/context.json" +SUITE_SPEC = SUITE_P4C_GEN + "/config.spec" + +SUITE_LOG = "log" + +SUITE_PATH_GENERATED = [SUITE_CONF_BF_SWITCHD, SUITE_P4C_GEN, SUITE_LOG] diff --git a/e2e-test/tools/run_bf_switchd.py b/e2e-test/tools/run_bf_switchd.py new file mode 100755 index 0000000..0335c03 --- /dev/null +++ b/e2e-test/tools/run_bf_switchd.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import string +from pathlib import Path +from subprocess import run + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + run_bf_switchd(args.test_dir) + + +def run_bf_switchd(test_dir, in_background=False): + util.assert_exists(test_dir) + + # Generate bf_switchd conf file + conf_template_path = config.THIS_DIR / "template" / config.SUITE_CONF_BF_SWITCHD + conf_template = string.Template(conf_template_path.read_text()) + # These paths have to be absolute, + # or they will be interpreted as relative to $SDE_INSTALL. + test_dir = test_dir.resolve() + conf = conf_template.substitute( + bfrt_config=test_dir / config.SUITE_BFRT, + context=test_dir / config.SUITE_CONTEXT, + config=test_dir / config.SUITE_SPEC, + ) + conf_path = test_dir / config.SUITE_CONF_BF_SWITCHD + conf_path.write_text(conf) + + # Configure hugepages for DPDK + sde_env = util.get_sde_env() + sde_install = sde_env["SDE_INSTALL"] + run(f"{sde_install}/bin/dpdk-hugepages.py -p 2M --setup 32M", shell=True) + run(f"{sde_install}/bin/dpdk-hugepages.py -s", shell=True) + + # Run the command + bin_name = "bf_switchd" + bin_path, log_dir, sde_env = util.bf_prepare(test_dir, bin_name) + cmd = [bin_path] + cmd += ["--install-dir", sde_env["SDE_INSTALL"]] + cmd += ["--conf-file", conf_path.resolve()] + return util.bf_run(cmd, log_dir, sde_env, in_background, with_sudo=True) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/run_bfshell.py b/e2e-test/tools/run_bfshell.py new file mode 100755 index 0000000..81b29d0 --- /dev/null +++ b/e2e-test/tools/run_bfshell.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import argparse +import logging +from pathlib import Path + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + run_bfshell(args.test_dir) + + +def run_bfshell(test_dir, in_background=False): + util.assert_exists(test_dir) + + # Run the command + bin_name = "bfshell" + bin_path, log_dir, sde_env = util.bf_prepare(test_dir, bin_name) + cmd = [bin_path] + cmd += ["-f", (test_dir / config.SUITE_CMDS_BFSHELL).resolve()] + return util.bf_run(cmd, log_dir, sde_env, in_background) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/template/conf_bf_switchd.json b/e2e-test/tools/template/conf_bf_switchd.json new file mode 100644 index 0000000..8c371ed --- /dev/null +++ b/e2e-test/tools/template/conf_bf_switchd.json @@ -0,0 +1,46 @@ +{ + "chip_list": [ + { + "id": "asic-0", + "chip_family": "dpdk", + "instance": 0 + } + ], + "instance": 0, + "p4_devices": [ + { + "device-id": 0, + "eal-args": "dummy -n 4 -c 3", + "mempools": [ + { + "name": "MEMPOOL0", + "buffer_size": 2304, + "pool_size": 1024, + "cache_size": 256, + "numa_node": 0 + } + ], + "p4_programs": [ + { + "program-name": "main", + "bfrt-config": "$bfrt_config", + "p4_pipelines": [ + { + "p4_pipeline_name": "pipe", + "core_id": 1, + "numa_node": 0, + "context": "$context", + "config": "$config", + "pipe_scope": [ + 0, + 1, + 2, + 3 + ] + } + ] + } + ] + } + ] +} diff --git a/e2e-test/tools/test.py b/e2e-test/tools/test.py new file mode 100755 index 0000000..0ff753b --- /dev/null +++ b/e2e-test/tools/test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import argparse +import atexit +import logging +import time +from pathlib import Path +from subprocess import run + +import config +import util +from clean import clean +from compile import compile +from run_bf_switchd import run_bf_switchd +from run_bfshell import run_bfshell + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + test(args.test_dir) + + +def test(test_dir): + util.assert_exists(test_dir) + + clean(test_dir) + compile(test_dir) + + # Run bf_switchd in background + # Wait until bfshell is up + p, stdout = run_bf_switchd(test_dir, in_background=True) + atexit.register(p.kill) + while not stdout.read_text().endswith("bfshell> "): + time.sleep(1) + + # Run some bfshell commands + # Wait until all commands are finished + p, stdout = run_bfshell(test_dir, in_background=True) + atexit.register(p.kill) + while "finished" not in stdout.read_text(): + time.sleep(1) + + # Run some shell commands + cmd = ["bash", test_dir / config.SUITE_CMDS_SHELL] + run(cmd, check=True) + + # Run the testing script + cmd = ["python3", test_dir / config.SUITE_TEST] + run(cmd) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/util.py b/e2e-test/tools/util.py new file mode 100644 index 0000000..b273d15 --- /dev/null +++ b/e2e-test/tools/util.py @@ -0,0 +1,89 @@ +import logging +import os +import shutil +from functools import lru_cache +from pathlib import Path +from subprocess import Popen, run + + +def assert_exists(path): + path = Path(path) + assert path.exists(), f"{path} doesn't exist" + + +def bf_prepare(test_dir, bin_name): + sde_env = get_sde_env() + sde_install = sde_env["SDE_INSTALL"] + bin_path = Path(f"{sde_install}/bin/{bin_name}") + assert bin_path.exists(), f"Required binary {bin_path} doesn't exist" + log_dir = test_dir / f"log/{bin_name}/" + log_dir.mkdir(parents=True, exist_ok=True) + return bin_path, log_dir, sde_env + + +def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): + if with_sudo: + # Passing environment variables this way is necessary. See + # https://github.com/Yi-Tseng/p4-dpdk-target-notes#start-the-switch + sudo = [ + "sudo", + "-E", + "PATH=" + sde_env["PATH"], + "LD_LIBRARY_PATH=" + sde_env["LD_LIBRARY_PATH"], + ] + cmd = sudo + cmd + log_cmd(cmd) + stdout = None + if in_background: + stdout = log_dir / "stdout.log" + stderr = log_dir / "stderr.log" + # Set bufsize to 0 because we need to check stdout in real time later + p = Popen( + cmd, + bufsize=0, + stdout=stdout.open("w"), + stderr=stderr.open("w"), + cwd=log_dir, + env=sde_env, + ) + else: + p = run(cmd, cwd=log_dir, env=sde_env, check=True) + logging.info(f"PID: {p.pid}") + return p, stdout + + +@lru_cache(maxsize=None) +def get_sde_env(): + sde_env = {} + sde_env.update(os.environ) + for env_var in ["SDE", "SDE_INSTALL"]: + assert env_var in sde_env, f"Environment variable {env_var} not set" + logging.info(f"Using {env_var}: {sde_env[env_var]}") + sde_install = sde_env["SDE_INSTALL"] + sde_env["LD_LIBRARY_PATH"] = f"{sde_install}/lib" + sde_env["LD_LIBRARY_PATH"] += f":{sde_install}/lib64" + sde_env["LD_LIBRARY_PATH"] += f":{sde_install}/lib/x86_64-linux-gnu" + sde_env["PYTHONPATH"] = f"{sde_install}/lib/python3.10" + sde_env["PYTHONPATH"] += f":{sde_install}/lib/python3.10/lib-dynload" + sde_env["PYTHONPATH"] += f":{sde_install}/lib/python3.10/site-packages" + sde_env["PYTHONHOME"] = f"{sde_install}/lib/python3.10" + return sde_env + + +def log_cmd(cmd): + if isinstance(cmd, list): + cmd = [arg.as_posix() if isinstance(arg, Path) else arg for arg in cmd] + cmd = " ".join(cmd) + logging.info(f"Run command:\n {cmd}\n") + + +def remove_file_or_dir(path): + path = Path(path) + + if not path.exists(): + return + + if path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) diff --git a/src/pipe_mgr/shared/dal/dpdk/dal_init.c b/src/pipe_mgr/shared/dal/dpdk/dal_init.c index 16fbb49..35a76a8 100644 --- a/src/pipe_mgr/shared/dal/dpdk/dal_init.c +++ b/src/pipe_mgr/shared/dal/dpdk/dal_init.c @@ -114,6 +114,7 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, snprintf(buffer, sizeof(buffer), "gcc -c -O3 -fpic " "-Wno-deprecated-declarations -o %s %s -I %s", o_filepath, c_filepath, i_filepath); + LOG_TRACE("Running command: %s\n", buffer); status = system(buffer); if (status) { LOG_ERROR("%s line:%d Cannot generate %s file\n", @@ -122,14 +123,25 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, } memset(buffer, 0, sizeof(buffer)); - snprintf(buffer, sizeof(buffer), "gcc -shared %s -o %s ", + snprintf(buffer, sizeof(buffer), "gcc -shared %s -o %s", o_filepath, so_filepath); + LOG_TRACE("Running command: %s\n", buffer); status = system(buffer); if (status) { + LOG_ERROR("%s line:%d Command (%s) failed with exit code: %d\n", + __func__, __LINE__, buffer, status); + } + // TODO: Ideally, we should check whether status is 0 below. But + // sometimes the returned status code is -1, even when the .so file is + // generated properly. Checking file existence is a temporary + // workaround. Need to further investigate and solve this issue. + fd = fopen(so_filepath, "r"); + if (!fd) { LOG_ERROR("%s line:%d Cannot generate %s file\n", __func__, __LINE__, so_filepath); return BF_INTERNAL_ERROR; } + fclose(fd); fd = fopen(IOSPEC_FILE_PATH, "r"); if (!fd) { diff --git a/tools/docker_build.sh b/tools/docker_build.sh new file mode 100755 index 0000000..5b7a027 --- /dev/null +++ b/tools/docker_build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# +# Build a Docker image for the P4 DPDK stack. + +IMAGE_NAME=p4lang/p4-dpdk-target + +docker build -t $IMAGE_NAME . diff --git a/tools/docker_exec_bash.sh b/tools/docker_exec_bash.sh new file mode 100755 index 0000000..b67f4b2 --- /dev/null +++ b/tools/docker_exec_bash.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# +# Run a new bash in a running container. + +CONTAINER_NAME=p4-dpdk-target + +docker exec -it $CONTAINER_NAME bash diff --git a/tools/docker_run_bash.sh b/tools/docker_run_bash.sh new file mode 100755 index 0000000..4606362 --- /dev/null +++ b/tools/docker_run_bash.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Run a bash in a new container. + +IMAGE_NAME=p4lang/p4-dpdk-target +CONTAINER_NAME=p4-dpdk-target +MOUNT_DIR=/home/work + +docker run -it --rm --privileged \ + --name $CONTAINER_NAME \ + -v /dev/hugepages:/dev/hugepages \ + -v "$PWD":$MOUNT_DIR -w $MOUNT_DIR \ + $IMAGE_NAME bash diff --git a/tools/docker_run_smoke_test.sh b/tools/docker_run_smoke_test.sh new file mode 100755 index 0000000..0e26ed6 --- /dev/null +++ b/tools/docker_run_smoke_test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Run the smoke test in a container. + +IMAGE_NAME=p4lang/p4-dpdk-target +CONTAINER_NAME=p4-dpdk-target +MOUNT_DIR=/home/work + +docker run -it --rm --privileged \ + --name $CONTAINER_NAME \ + -v /dev/hugepages:/dev/hugepages \ + -v "$PWD":$MOUNT_DIR -w $MOUNT_DIR \ + $IMAGE_NAME \ + python3 e2e-test/tools/test.py e2e-test/suites/pna/basicfwd diff --git a/tools/setup/install_dep.py b/tools/setup/install_dep.py index cd772a2..bf09294 100644 --- a/tools/setup/install_dep.py +++ b/tools/setup/install_dep.py @@ -140,7 +140,7 @@ def execute_system_command(command: List[str], ## setup proxies for pip to run for item in pip_packages: - pip3_install_command = ["pip3", "install", item] + pip3_install_command = ["pip3", "install", "--upgrade", item] #pip_install_command = ["pip", "install", item] #pip2_install_command = ["pip2", "install", item] print (execute_system_command (pip3_install_command)[0]) diff --git a/zlog-cfg b/zlog-cfg index c0716d0..04e6fa9 100644 --- a/zlog-cfg +++ b/zlog-cfg @@ -23,7 +23,7 @@ BF_LLD.ERROR >stdout;console_format BF_LLD.DEBUG "p4_driver.log", 5M * 5 ;file_format BF_PIPE.ERROR >stdout;console_format -BF_PIPE.DEBUG "p4_driver.log", 5M * 5 ;file_format +BF_PIPE.ERROR "p4_driver.log", 5M * 5 ;file_format BF_DVM.ERROR >stdout;console_format BF_DVM.DEBUG "p4_driver.log", 5M * 5 ;file_format @@ -41,7 +41,7 @@ BF_PM.ERROR >stdout;console_format BF_PM.DEBUG "p4_driver.log", 5M * 5 ;file_format BF_BFRT.ERROR >stdout;console_format -BF_BFRT.DEBUG "p4_driver.log", 5M * 5 ;file_format +BF_BFRT.ERROR "p4_driver.log", 5M * 5 ;file_format KRNLMON.ERROR >stdout;console_format KRNLMON.DEBUG "krnlmon.log", 5M * 5 ;file_format