From 4a66f2ae997d0ff4185c586b218fb26060c74156 Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Tue, 23 May 2023 16:53:28 +0000 Subject: [PATCH 01/14] Add a Dockerfile for a reproducible environment The final image is 5.3GB. That seems too large. There should be room for optimization. --- Dockerfile | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..96c40ba --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +ARG BASE_IMAGE=p4lang/p4c:latest +FROM ${BASE_IMAGE} +LABEL maintainer="P4 Developers " + +# Install some packages needed for later steps +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y \ + python-is-python3 \ + python3 \ + python3-pip \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +# Set environment variables +# We don't simply source tools/setup/p4sde_env_setup.sh, +# because we want to persist these in the running container. +ENV SDE=/home/sde +ENV SDE_INSTALL=${SDE}/install +ENV LD_LIBRARY_PATH=${SDE_INSTALL}/lib:${SDE_INSTALL}/lib64:${SDE_INSTALL}/lib/x86_64-linux-gnu/ +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64 +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib + +# Copy repo files in +ARG REPO_NAME=p4-dpdk-target +COPY . ${SDE}/${REPO_NAME}/ + +# Install dependent packages +WORKDIR ${SDE}/${REPO_NAME}/tools/setup +RUN pip3 install distro \ + && apt-get update \ + && python3 install_dep.py \ + && rm -rf /var/lib/apt/lists/* + +# Build P4 DPDK target +WORKDIR ${SDE}/${REPO_NAME} +RUN ./autogen.sh \ + && ./configure --prefix=${SDE_INSTALL} \ + && make -j \ + && make install + +# Set working directory +WORKDIR ${SDE} From 3f84d18d96a5d95da52cb6701e647d66f63fbada Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Tue, 30 May 2023 21:27:23 +0000 Subject: [PATCH 02/14] Add a simple testing pipeline. The testing scripts are not finished yet. But I ran into some issues that might need to be resolved before I can move on. --- .dockerignore | 9 ++ Dockerfile | 1 + docker-build.sh | 5 + docker-run.sh | 9 ++ e2e-test/.gitignore | 2 + e2e-test/pna/simple_l2_forwarding/README.md | 27 ++++++ .../simple_l2_forwarding/bf_switchd_conf.json | 47 ++++++++++ .../pna/simple_l2_forwarding/cmd/build.sh | 14 +++ .../pna/simple_l2_forwarding/cmd/clean.sh | 3 + .../cmd/run_bf_switchd.sh | 39 ++++++++ e2e-test/pna/simple_l2_forwarding/pipe.p4 | 93 +++++++++++++++++++ 11 files changed, 249 insertions(+) create mode 100644 .dockerignore create mode 100755 docker-build.sh create mode 100755 docker-run.sh create mode 100644 e2e-test/.gitignore create mode 100644 e2e-test/pna/simple_l2_forwarding/README.md create mode 100644 e2e-test/pna/simple_l2_forwarding/bf_switchd_conf.json create mode 100755 e2e-test/pna/simple_l2_forwarding/cmd/build.sh create mode 100755 e2e-test/pna/simple_l2_forwarding/cmd/clean.sh create mode 100755 e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh create mode 100644 e2e-test/pna/simple_l2_forwarding/pipe.p4 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bb5a071 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +**/.git** +**/Dockerfile +**/README +**/*.md + +.github +docker* +e2e-test +examples diff --git a/Dockerfile b/Dockerfile index 96c40ba..c96050e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ ENV SDE_INSTALL=${SDE}/install ENV LD_LIBRARY_PATH=${SDE_INSTALL}/lib:${SDE_INSTALL}/lib64:${SDE_INSTALL}/lib/x86_64-linux-gnu/ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64 ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib +ENV PATH=${SDE_INSTALL}/bin:${PATH} # Copy repo files in ARG REPO_NAME=p4-dpdk-target diff --git a/docker-build.sh b/docker-build.sh new file mode 100755 index 0000000..8019dab --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +DOCKER_IMAGE=p4-dpdk-target + +docker build -t $DOCKER_IMAGE . diff --git a/docker-run.sh b/docker-run.sh new file mode 100755 index 0000000..0596027 --- /dev/null +++ b/docker-run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +DOCKER_IMAGE=p4-dpdk-target +DOCKER_MOUNTDIR=/home/work + +docker run -it --rm --privileged \ + -v /dev/hugepages:/dev/hugepages \ + -v "$PWD":$DOCKER_MOUNTDIR -w $DOCKER_MOUNTDIR \ + $DOCKER_IMAGE bash diff --git a/e2e-test/.gitignore b/e2e-test/.gitignore new file mode 100644 index 0000000..dad7815 --- /dev/null +++ b/e2e-test/.gitignore @@ -0,0 +1,2 @@ +pipe/ +log/ diff --git a/e2e-test/pna/simple_l2_forwarding/README.md b/e2e-test/pna/simple_l2_forwarding/README.md new file mode 100644 index 0000000..7cc1949 --- /dev/null +++ b/e2e-test/pna/simple_l2_forwarding/README.md @@ -0,0 +1,27 @@ +# Simple L2 Forwarding Test + +## Environment setup + +If you have set up a local environment already, you can skip this part. Otherwise, you can also use a Docker container to run this test. + +The following commands need to run at the root of this repo where the `Dockerfile` resides. + +First build a Docker image: +``` +./docker-build.sh +``` + +Then run a Docker container from this image and enter an interactive shell: +``` +./docker-run.sh +``` + +## Manual testing steps + +Switch to this directory and run the following commands: +``` +./cmd/build.sh +./cmd/run_bf_switchd.sh +``` + +Then you shall see the `bfshell>` prompt. diff --git a/e2e-test/pna/simple_l2_forwarding/bf_switchd_conf.json b/e2e-test/pna/simple_l2_forwarding/bf_switchd_conf.json new file mode 100644 index 0000000..1e38bd2 --- /dev/null +++ b/e2e-test/pna/simple_l2_forwarding/bf_switchd_conf.json @@ -0,0 +1,47 @@ +{ + "chip_list": [ + { + "id": "asic-0", + "chip_family": "dpdk", + "instance": 0 + } + ], + "instance": 0, + "p4_devices": [ + { + "device-id": 0, + "eal-args": "dummy -n 4 -c 7", + "mempools": [ + { + "name": "MEMPOOL0", + "buffer_size": 2304, + "pool_size": 1024, + "cache_size": 256, + "numa_node": 0 + } + ], + "p4_programs": [ + { + "program-name": "simple_l2_forwarding", + "bfrt-config": "share/e2e-test/pna/simple_l2_forwarding/pipe/tdi.json", + "p4_pipelines": [ + { + "p4_pipeline_name": "pipe", + "core_id": 1, + "numa_node": 0, + "context": "share/e2e-test/pna/simple_l2_forwarding/pipe/context.json", + "config": "share/e2e-test/pna/simple_l2_forwarding/pipe/config.spec", + "pipe_scope": [ + 0, + 1, + 2, + 3 + ], + "path": "share/e2e-test/pna/simple_l2_forwarding" + } + ] + } + ] + } + ] +} diff --git a/e2e-test/pna/simple_l2_forwarding/cmd/build.sh b/e2e-test/pna/simple_l2_forwarding/cmd/build.sh new file mode 100755 index 0000000..32d223d --- /dev/null +++ b/e2e-test/pna/simple_l2_forwarding/cmd/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +OUTPUT_DIR=pipe + +rm -rf $OUTPUT_DIR +mkdir -p $OUTPUT_DIR + +p4c-dpdk pipe.p4 \ + --arch pna \ + --p4runtime-files $OUTPUT_DIR/p4info.txt \ + --bf-rt-schema $OUTPUT_DIR/bfrt.json \ + --tdi $OUTPUT_DIR/tdi.json \ + --context $OUTPUT_DIR/context.json \ + -o $OUTPUT_DIR/config.spec diff --git a/e2e-test/pna/simple_l2_forwarding/cmd/clean.sh b/e2e-test/pna/simple_l2_forwarding/cmd/clean.sh new file mode 100755 index 0000000..8c0f76f --- /dev/null +++ b/e2e-test/pna/simple_l2_forwarding/cmd/clean.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +rm -rf log pipe diff --git a/e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh b/e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh new file mode 100755 index 0000000..2c9733b --- /dev/null +++ b/e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# https://github.com/p4lang/p4-dpdk-target#running-reference-app +set -e + +# Per test settings +TEST_NAME=pna/simple_l2_forwarding +TEST_DIR="$PWD" + +# Make sure SDE environment variables are set +[ -z "$SDE" ] && echo "Environment variable SDE not set" && exit 1 +[ -z "$SDE_INSTALL" ] && echo "Environment variable SDE_INSTALL not set" && exit 1 +echo "Using SDE: $SDE" +echo "Using SDE_INSTALL: $SDE_INSTALL" + +# Configure hugepages for DPDK +# This has to be done before switching to the SDE environment. +dpdk-hugepages.py -p 1G --setup 2G +dpdk-hugepages.py -s + +# Switch to the SDE environment +export LD_LIBRARY_PATH="$SDE_INSTALL/lib:$SDE_INSTALL/lib64:$SDE_INSTALL/lib/x86_64-linux-gnu" +export PYTHONPATH="$SDE_INSTALL/lib/python3.10/:$SDE_INSTALL/lib/python3.10/lib-dynload:$SDE_INSTALL/lib/python3.10/site-packages" +export PYTHONHOME="$SDE_INSTALL/lib/python3.10" + +# All files referred to in the bf_switchd conf file (pipe/config.spec) have to be under $SDE_INSTALL, +# so let's copy them over. +SDE_CONF_DIR="$SDE_INSTALL/share/e2e-test/$TEST_NAME" +rm -rf "$SDE_CONF_DIR" +mkdir -p "$SDE_CONF_DIR" +cp -r "$TEST_DIR/pipe" "$SDE_CONF_DIR/pipe" + +# Run bf_switchd in the log directory to collect all log files there +CONF_FILE="$TEST_DIR/bf_switchd_conf.json" +LOG_DIR="$TEST_DIR/log/bf_switchd" +mkdir -p "$LOG_DIR" +cd "$LOG_DIR" +"$SDE_INSTALL/bin/bf_switchd" \ + --install-dir "$SDE_INSTALL" \ + --conf-file "$CONF_FILE" diff --git a/e2e-test/pna/simple_l2_forwarding/pipe.p4 b/e2e-test/pna/simple_l2_forwarding/pipe.p4 new file mode 100644 index 0000000..e414165 --- /dev/null +++ b/e2e-test/pna/simple_l2_forwarding/pipe.p4 @@ -0,0 +1,93 @@ +#include +#include + +header eth_h { + bit<48> dst_addr; + bit<48> src_addr; + bit<16> ether_type; +} + +header custom_h { + bit<32> marker; +} + +struct header_t { + eth_h eth; + custom_h custom; +} + +struct metadata_t {} + +parser MainParser( + packet_in pkt, + out header_t hdr, + inout metadata_t meta, + in pna_main_parser_input_metadata_t istd +) { + state start { + pkt.extract(hdr.eth); + pkt.extract(hdr.custom); + transition accept; + } +} + +control PreControl( + in header_t hdr, + inout metadata_t meta, + in pna_pre_input_metadata_t istd, + inout pna_pre_output_metadata_t ostd +) { + apply {} +} + +control MainControl( + inout header_t hdr, + inout metadata_t meta, + in pna_main_input_metadata_t istd, + inout pna_main_output_metadata_t ostd +) { + action mark_and_forward( + bit<32> marker, + PortId_t port + ) { + hdr.custom.marker = marker; + send_to_port(port); + } + + action drop() { + drop_packet(); + } + + table forwarding { + key = { + hdr.eth.dst_addr: exact; + } + actions = { + mark_and_forward; + drop; + } + const default_action = drop(); + } + + apply { + forwarding.apply(); + } +} + +control MainDeparser( + packet_out pkt, + in header_t hdr, + in metadata_t meta, + in pna_main_output_metadata_t ostd +) { + apply { + pkt.emit(hdr); + } +} + +PNA_NIC( + MainParser(), + PreControl(), + MainControl(), + MainDeparser() +) main; From 8aa4e4387cd436613bd11e397946c3b4a8758d7f Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Wed, 31 May 2023 16:27:54 +0000 Subject: [PATCH 03/14] Update TDI and DPDK submodules. TDI is updated to the latest commit (on Mar 1, 2023). DPDK is updated to the latest release (v23.03). --- src/lld/dpdk/dpdk_src | 2 +- third-party/tdi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lld/dpdk/dpdk_src b/src/lld/dpdk/dpdk_src index baf13c3..189c02f 160000 --- a/src/lld/dpdk/dpdk_src +++ b/src/lld/dpdk/dpdk_src @@ -1 +1 @@ -Subproject commit baf13c3135d0c5998fff7edc23fb89412dc89246 +Subproject commit 189c02f5ec891ed02927062e124e8ca03adf74e7 diff --git a/third-party/tdi b/third-party/tdi index 2f9ce92..fe1308c 160000 --- a/third-party/tdi +++ b/third-party/tdi @@ -1 +1 @@ -Subproject commit 2f9ce92cd3e2514847faa03157e715ac7b49dc74 +Subproject commit fe1308cae6f4e6e673172d17215494987922bee6 From f52793c229ddc74d4b14d506f3ff92539623fc4f Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Fri, 2 Jun 2023 21:36:42 +0000 Subject: [PATCH 04/14] Convert ad-hoc shell scripts into reusable Python scripts --- e2e-test/.gitignore | 2 - e2e-test/README.md | 52 ++++++ e2e-test/pna/simple_l2_forwarding/README.md | 27 ---- .../pna/simple_l2_forwarding/cmd/build.sh | 14 -- .../pna/simple_l2_forwarding/cmd/clean.sh | 3 - .../cmd/run_bf_switchd.sh | 39 ----- e2e-test/suites/.gitignore | 3 + .../pna/simple_l2_forwarding/main.p4} | 0 e2e-test/tools/.gitignore | 1 + e2e-test/tools/cmd.py | 152 ++++++++++++++++++ .../template}/bf_switchd_conf.json | 11 +- e2e-test/tools/util.py | 39 +++++ 12 files changed, 252 insertions(+), 91 deletions(-) delete mode 100644 e2e-test/.gitignore create mode 100644 e2e-test/README.md delete mode 100644 e2e-test/pna/simple_l2_forwarding/README.md delete mode 100755 e2e-test/pna/simple_l2_forwarding/cmd/build.sh delete mode 100755 e2e-test/pna/simple_l2_forwarding/cmd/clean.sh delete mode 100755 e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh create mode 100644 e2e-test/suites/.gitignore rename e2e-test/{pna/simple_l2_forwarding/pipe.p4 => suites/pna/simple_l2_forwarding/main.p4} (100%) create mode 100644 e2e-test/tools/.gitignore create mode 100755 e2e-test/tools/cmd.py rename e2e-test/{pna/simple_l2_forwarding => tools/template}/bf_switchd_conf.json (69%) create mode 100644 e2e-test/tools/util.py diff --git a/e2e-test/.gitignore b/e2e-test/.gitignore deleted file mode 100644 index dad7815..0000000 --- a/e2e-test/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -pipe/ -log/ diff --git a/e2e-test/README.md b/e2e-test/README.md new file mode 100644 index 0000000..4fc9c70 --- /dev/null +++ b/e2e-test/README.md @@ -0,0 +1,52 @@ +# P4 DPDK Target End-to-End Testing + +## Directory structure + +``` +suites/ + Collection of test suites. + + // + Individual test suite. + + (generated) log/ + Log files generated during testing. + (generated) p4c_gen/ + P4 compiler outputs generated from main.p4. + (generated) bf_switchd_config.json: + Config file needed by the bf_switchd binary. + main.p4: + The P4 pipeline code. + +tools/ + Tools used to run the tests. +``` + +## Example testing workflow + +### Compile P4 code + +```console +./tools/cmd.py compile suites/pna/simple_l2_forwarding +``` + +The following paths will be generated: + +- `suites/pna/simple_l2_forwarding/build/` + +### Run `bf_switchd` + +```console +./tools/cmd.py bf_switchd suites/pna/simple_l2_forwarding +``` + +The following paths will be generated: + +- `suites/pna/simple_l2_forwarding/log/bf_switchd/` +- `suites/pna/simple_l2_forwarding/bf_switchd_conf.json` + +### Clean up (optional) + +```console +./tools/cmd.py clean suites/pna/simple_l2_forwarding +``` diff --git a/e2e-test/pna/simple_l2_forwarding/README.md b/e2e-test/pna/simple_l2_forwarding/README.md deleted file mode 100644 index 7cc1949..0000000 --- a/e2e-test/pna/simple_l2_forwarding/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Simple L2 Forwarding Test - -## Environment setup - -If you have set up a local environment already, you can skip this part. Otherwise, you can also use a Docker container to run this test. - -The following commands need to run at the root of this repo where the `Dockerfile` resides. - -First build a Docker image: -``` -./docker-build.sh -``` - -Then run a Docker container from this image and enter an interactive shell: -``` -./docker-run.sh -``` - -## Manual testing steps - -Switch to this directory and run the following commands: -``` -./cmd/build.sh -./cmd/run_bf_switchd.sh -``` - -Then you shall see the `bfshell>` prompt. diff --git a/e2e-test/pna/simple_l2_forwarding/cmd/build.sh b/e2e-test/pna/simple_l2_forwarding/cmd/build.sh deleted file mode 100755 index 32d223d..0000000 --- a/e2e-test/pna/simple_l2_forwarding/cmd/build.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -OUTPUT_DIR=pipe - -rm -rf $OUTPUT_DIR -mkdir -p $OUTPUT_DIR - -p4c-dpdk pipe.p4 \ - --arch pna \ - --p4runtime-files $OUTPUT_DIR/p4info.txt \ - --bf-rt-schema $OUTPUT_DIR/bfrt.json \ - --tdi $OUTPUT_DIR/tdi.json \ - --context $OUTPUT_DIR/context.json \ - -o $OUTPUT_DIR/config.spec diff --git a/e2e-test/pna/simple_l2_forwarding/cmd/clean.sh b/e2e-test/pna/simple_l2_forwarding/cmd/clean.sh deleted file mode 100755 index 8c0f76f..0000000 --- a/e2e-test/pna/simple_l2_forwarding/cmd/clean.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -rm -rf log pipe diff --git a/e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh b/e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh deleted file mode 100755 index 2c9733b..0000000 --- a/e2e-test/pna/simple_l2_forwarding/cmd/run_bf_switchd.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# https://github.com/p4lang/p4-dpdk-target#running-reference-app -set -e - -# Per test settings -TEST_NAME=pna/simple_l2_forwarding -TEST_DIR="$PWD" - -# Make sure SDE environment variables are set -[ -z "$SDE" ] && echo "Environment variable SDE not set" && exit 1 -[ -z "$SDE_INSTALL" ] && echo "Environment variable SDE_INSTALL not set" && exit 1 -echo "Using SDE: $SDE" -echo "Using SDE_INSTALL: $SDE_INSTALL" - -# Configure hugepages for DPDK -# This has to be done before switching to the SDE environment. -dpdk-hugepages.py -p 1G --setup 2G -dpdk-hugepages.py -s - -# Switch to the SDE environment -export LD_LIBRARY_PATH="$SDE_INSTALL/lib:$SDE_INSTALL/lib64:$SDE_INSTALL/lib/x86_64-linux-gnu" -export PYTHONPATH="$SDE_INSTALL/lib/python3.10/:$SDE_INSTALL/lib/python3.10/lib-dynload:$SDE_INSTALL/lib/python3.10/site-packages" -export PYTHONHOME="$SDE_INSTALL/lib/python3.10" - -# All files referred to in the bf_switchd conf file (pipe/config.spec) have to be under $SDE_INSTALL, -# so let's copy them over. -SDE_CONF_DIR="$SDE_INSTALL/share/e2e-test/$TEST_NAME" -rm -rf "$SDE_CONF_DIR" -mkdir -p "$SDE_CONF_DIR" -cp -r "$TEST_DIR/pipe" "$SDE_CONF_DIR/pipe" - -# Run bf_switchd in the log directory to collect all log files there -CONF_FILE="$TEST_DIR/bf_switchd_conf.json" -LOG_DIR="$TEST_DIR/log/bf_switchd" -mkdir -p "$LOG_DIR" -cd "$LOG_DIR" -"$SDE_INSTALL/bin/bf_switchd" \ - --install-dir "$SDE_INSTALL" \ - --conf-file "$CONF_FILE" diff --git a/e2e-test/suites/.gitignore b/e2e-test/suites/.gitignore new file mode 100644 index 0000000..23aec05 --- /dev/null +++ b/e2e-test/suites/.gitignore @@ -0,0 +1,3 @@ +log/ +p4c_gen/ +bf_switchd_conf.json diff --git a/e2e-test/pna/simple_l2_forwarding/pipe.p4 b/e2e-test/suites/pna/simple_l2_forwarding/main.p4 similarity index 100% rename from e2e-test/pna/simple_l2_forwarding/pipe.p4 rename to e2e-test/suites/pna/simple_l2_forwarding/main.p4 diff --git a/e2e-test/tools/.gitignore b/e2e-test/tools/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/e2e-test/tools/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/e2e-test/tools/cmd.py b/e2e-test/tools/cmd.py new file mode 100755 index 0000000..473927f --- /dev/null +++ b/e2e-test/tools/cmd.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import string +from pathlib import Path +from subprocess import run + +import util + +THIS_DIR = Path(__file__).resolve().parent + +# Test suite path conventions + +SUITE_MAIN_P4 = "main.p4" +SUITE_BF_SWITCHD_CONF = "bf_switchd_conf.json" + +SUITE_P4C_GEN = "p4c_gen" +SUITE_P4INFO = SUITE_P4C_GEN + "/p4info.txt" +SUITE_BFRT = SUITE_P4C_GEN + "/bfrt.json" +SUITE_TDI = SUITE_P4C_GEN + "/tdi.json" +SUITE_CONTEXT = SUITE_P4C_GEN + "/context.json" +SUITE_SPEC = SUITE_P4C_GEN + "/config.spec" + +SUITE_LOG = "log" + +SUITE_PATH_GENERATED = [SUITE_BF_SWITCHD_CONF, SUITE_P4C_GEN, SUITE_LOG] + + +# Main entry point +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(required=True) + + add_parser_for_compile(subparsers) + add_parser_for_bf_switchd(subparsers) + add_parser_for_clean(subparsers) + + args = parser.parse_args() + args.func(args) + + +# The compile command + + +def add_parser_for_compile(subparsers): + parser = subparsers.add_parser("compile") + parser.add_argument("test_dir", type=Path) + parser.set_defaults(func=compile) + + +def compile(args): + test_dir = args.test_dir + + # Make sure test dir exists + assert test_dir.exists(), f"{test_dir} doesn't exist" + + # Prepare compiler output dir + output_dir = test_dir / SUITE_P4C_GEN + if output_dir.exists(): + logging.info(f"Removing existing compiler output dir {output_dir}") + util.remove_file_or_dir(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Compile P4 source + arch = test_dir.parent.stem + test_dir_prefix = test_dir.as_posix() + "/" + p4_source = test_dir_prefix + SUITE_MAIN_P4 + cmd = ["p4c-dpdk"] + cmd += ["--arch", arch] + cmd += ["--p4runtime-files", test_dir_prefix + SUITE_P4INFO] + cmd += ["--bf-rt-schema", test_dir_prefix + SUITE_BFRT] + cmd += ["--tdi", test_dir_prefix + SUITE_TDI] + cmd += ["--context", test_dir_prefix + SUITE_CONTEXT] + cmd += ["-o", test_dir_prefix + SUITE_SPEC] + cmd += [p4_source] + logging.info(f"Compiling {p4_source}") + util.log_cmd(cmd) + run(cmd, check=True) + logging.info("Compiling succeeds") + + +# The bf_switchd command + + +def add_parser_for_bf_switchd(subparsers): + parser = subparsers.add_parser("bf_switchd") + parser.add_argument("test_dir", type=Path) + parser.set_defaults(func=bf_switchd) + + +def bf_switchd(args): + test_dir = args.test_dir + + # Make sure test dir exists + assert test_dir.exists(), f"{test_dir} doesn't exist" + + # Generate bf_switchd conf file + conf_template_path = THIS_DIR / "template" / SUITE_BF_SWITCHD_CONF + conf_template = string.Template(conf_template_path.read_text()) + # These paths have to be absolute, + # or they will be interpreted as relative to $SDE_INSTALL + test_dir_prefix = test_dir.resolve().as_posix() + "/" + conf = conf_template.substitute( + bfrt_config=test_dir_prefix + SUITE_BFRT, + context=test_dir_prefix + SUITE_CONTEXT, + config=test_dir_prefix + SUITE_SPEC, + ) + conf_path = test_dir / SUITE_BF_SWITCHD_CONF + conf_path.write_text(conf) + + # Configure hugepages for DPDK + run("dpdk-hugepages.py -p 1G --setup 2G", shell=True, check=True) + run("dpdk-hugepages.py -s", shell=True, check=True) + + # Run bf_switchd + # Have to switch to the SDE environment. + sde_env = util.get_sde_env() + sde_install = sde_env["SDE_INSTALL"] + bin = f"{sde_install}/bin/bf_switchd" + assert Path(bin).exists(), f"Required binary {bin} doesn't exist" + log_dir = test_dir / "log/bf_switchd/" + log_dir.mkdir(parents=True, exist_ok=True) + cmd = [bin] + cmd += ["--install-dir", sde_env["SDE_INSTALL"]] + cmd += ["--conf-file", conf_path.resolve().as_posix()] + util.log_cmd(cmd) + run(cmd, cwd=log_dir, env=sde_env, check=True) + + +# The clean command + + +def add_parser_for_clean(subparsers): + parser = subparsers.add_parser("clean") + parser.add_argument("test_dir") + parser.set_defaults(func=clean) + + +def clean(args): + test_dir = Path(args.test_dir) + + for path in SUITE_PATH_GENERATED: + util.remove_file_or_dir(test_dir / path) + + +# Start + +if __name__ == "__main__": + main() diff --git a/e2e-test/pna/simple_l2_forwarding/bf_switchd_conf.json b/e2e-test/tools/template/bf_switchd_conf.json similarity index 69% rename from e2e-test/pna/simple_l2_forwarding/bf_switchd_conf.json rename to e2e-test/tools/template/bf_switchd_conf.json index 1e38bd2..677027d 100644 --- a/e2e-test/pna/simple_l2_forwarding/bf_switchd_conf.json +++ b/e2e-test/tools/template/bf_switchd_conf.json @@ -22,22 +22,21 @@ ], "p4_programs": [ { - "program-name": "simple_l2_forwarding", - "bfrt-config": "share/e2e-test/pna/simple_l2_forwarding/pipe/tdi.json", + "program-name": "main", + "bfrt-config": "$bfrt_config", "p4_pipelines": [ { "p4_pipeline_name": "pipe", "core_id": 1, "numa_node": 0, - "context": "share/e2e-test/pna/simple_l2_forwarding/pipe/context.json", - "config": "share/e2e-test/pna/simple_l2_forwarding/pipe/config.spec", + "context": "$context", + "config": "$config", "pipe_scope": [ 0, 1, 2, 3 - ], - "path": "share/e2e-test/pna/simple_l2_forwarding" + ] } ] } diff --git a/e2e-test/tools/util.py b/e2e-test/tools/util.py new file mode 100644 index 0000000..c55e9f7 --- /dev/null +++ b/e2e-test/tools/util.py @@ -0,0 +1,39 @@ +import logging +import os +import shutil +from pathlib import Path + + +def get_sde_env(): + sde_env = {} + sde_env.update(os.environ) + for env_var in ["SDE", "SDE_INSTALL"]: + assert env_var in sde_env, f"Environment variable {env_var} not set" + print(f"Using {env_var}: {sde_env[env_var]}") + sde_install = sde_env["SDE_INSTALL"] + sde_env["LD_LIBRARY_PATH"] = f"{sde_install}/lib" + sde_env["LD_LIBRARY_PATH"] += f":{sde_install}/lib64" + sde_env["LD_LIBRARY_PATH"] += f":{sde_install}/lib/x86_64-linux-gnu" + sde_env["PYTHONPATH"] = f"{sde_install}/lib/python3.10" + sde_env["PYTHONPATH"] += f":{sde_install}/lib/python3.10/lib-dynload" + sde_env["PYTHONPATH"] += f":{sde_install}/lib/python3.10/site-packages" + sde_env["PYTHONHOME"] = f"{sde_install}/lib/python3.10" + return sde_env + + +def log_cmd(cmd): + if isinstance(cmd, list): + cmd = " ".join(cmd) + logging.info(f"Run command: {cmd}") + + +def remove_file_or_dir(path): + path = Path(path) + + if not path.exists(): + return + + if path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) From 610dd88695928ed5ee42ce17e51b0abd8d983e3d Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Fri, 2 Jun 2023 23:49:13 +0000 Subject: [PATCH 05/14] Add the bfshell command --- docker-join.sh | 5 +++ docker-run.sh | 2 ++ e2e-test/README.md | 10 ++++++ .../pna/simple_l2_forwarding/bfshell_cmds.py | 34 +++++++++++++++++++ e2e-test/tools/cmd.py | 28 +++++++++++++++ 5 files changed, 79 insertions(+) create mode 100755 docker-join.sh create mode 100644 e2e-test/suites/pna/simple_l2_forwarding/bfshell_cmds.py diff --git a/docker-join.sh b/docker-join.sh new file mode 100755 index 0000000..d0429df --- /dev/null +++ b/docker-join.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +DOCKER_CONTAINER=p4-dpdk-target + +docker exec -it $DOCKER_CONTAINER bash diff --git a/docker-run.sh b/docker-run.sh index 0596027..e9620c9 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash DOCKER_IMAGE=p4-dpdk-target +DOCKER_CONTAINER=$DOCKER_IMAGE DOCKER_MOUNTDIR=/home/work docker run -it --rm --privileged \ + --name $DOCKER_CONTAINER \ -v /dev/hugepages:/dev/hugepages \ -v "$PWD":$DOCKER_MOUNTDIR -w $DOCKER_MOUNTDIR \ $DOCKER_IMAGE bash diff --git a/e2e-test/README.md b/e2e-test/README.md index 4fc9c70..dc25cb8 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -15,6 +15,8 @@ suites/ P4 compiler outputs generated from main.p4. (generated) bf_switchd_config.json: Config file needed by the bf_switchd binary. + bfshell_cmds.py: + Commands to run with bfshell. main.p4: The P4 pipeline code. @@ -45,6 +47,14 @@ The following paths will be generated: - `suites/pna/simple_l2_forwarding/log/bf_switchd/` - `suites/pna/simple_l2_forwarding/bf_switchd_conf.json` +### Run `bfshell` + +Keep the `bf_switchd` terminal open. In another terminal, run: + +```console +./tools/cmd.py bfshell suites/pna/simple_l2_forwarding +``` + ### Clean up (optional) ```console diff --git a/e2e-test/suites/pna/simple_l2_forwarding/bfshell_cmds.py b/e2e-test/suites/pna/simple_l2_forwarding/bfshell_cmds.py new file mode 100644 index 0000000..6ff369a --- /dev/null +++ b/e2e-test/suites/pna/simple_l2_forwarding/bfshell_cmds.py @@ -0,0 +1,34 @@ +tdi_python + +# Add ports + +for port_id in range(4): + tdi.port.port.add( + DEV_PORT=port_id, + PORT_TYPE="BF_DPDK_TAP", + PORT_DIR="PM_PORT_DIR_DEFAULT", + PORT_IN_ID=port_id, + PORT_OUT_ID =port_id, + PIPE_IN="pipe", + PIPE_OUT="pipe", + MEMPOOL="MEMPOOL0", + PORT_NAME=f"TAP{port_id}", + MTU=1500 + ) + +# Add entries + +from netaddr import IPAddress + +control = tdi.main.pipe.MainControl +table = control.forwarding +table.add_with_mark_and_forward( + dst_addr=IPAddress('192.168.1.101'), + marker=0x01, + port=1 +) +table.add_with_mark_and_forward( + dst_addr=IPAddress('192.168.1.102'), + marker=0x02, + port=2 +) diff --git a/e2e-test/tools/cmd.py b/e2e-test/tools/cmd.py index 473927f..766aa73 100755 --- a/e2e-test/tools/cmd.py +++ b/e2e-test/tools/cmd.py @@ -14,6 +14,7 @@ SUITE_MAIN_P4 = "main.p4" SUITE_BF_SWITCHD_CONF = "bf_switchd_conf.json" +SUITE_BFSHELL_CMDS = "bfshell_cmds.py" SUITE_P4C_GEN = "p4c_gen" SUITE_P4INFO = SUITE_P4C_GEN + "/p4info.txt" @@ -36,6 +37,7 @@ def main(): add_parser_for_compile(subparsers) add_parser_for_bf_switchd(subparsers) + add_parser_for_bfshell(subparsers) add_parser_for_clean(subparsers) args = parser.parse_args() @@ -130,6 +132,32 @@ def bf_switchd(args): run(cmd, cwd=log_dir, env=sde_env, check=True) +# The bfshell command + + +def add_parser_for_bfshell(subparsers): + parser = subparsers.add_parser("bfshell") + parser.add_argument("test_dir", type=Path) + parser.set_defaults(func=bfshell) + + +def bfshell(args): + test_dir = args.test_dir + + # Make sure test dir exists + assert test_dir.exists(), f"{test_dir} doesn't exist" + + # Run bfshell + # Have to switch to the SDE environment. + sde_env = util.get_sde_env() + sde_install = sde_env["SDE_INSTALL"] + bin = f"{sde_install}/bin/bfshell" + assert Path(bin).exists(), f"Required binary {bin} doesn't exist" + cmd = [bin] + cmd += ["-f", (test_dir / SUITE_BFSHELL_CMDS).resolve().as_posix()] + run(cmd, env=sde_env, check=True) + + # The clean command From a056ba25b9a1d50ddc3f7ba9702b02f04ae8b02a Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Wed, 7 Jun 2023 17:27:20 +0000 Subject: [PATCH 06/14] Set up a basic automated smoke test There are still lots of issues to solve before this smoke test becomes meaningful enough. But this could be a good starting point. --- .dockerignore | 2 + Dockerfile | 1 - docker-build.sh | 5 - docker-join.sh | 5 - docker-run.sh | 11 -- e2e-test/README.md | 75 +++++++- e2e-test/suites/pna/basicfwd/README.md | 3 + .../cmds_bfshell.py} | 20 +- e2e-test/suites/pna/basicfwd/cmds_shell.sh | 11 ++ .../main.p4 | 36 +--- e2e-test/suites/pna/basicfwd/test.py | 42 ++++ e2e-test/tools/clean.py | 29 +++ e2e-test/tools/cmd.py | 180 ------------------ e2e-test/tools/compile.py | 48 +++++ e2e-test/tools/config.py | 22 +++ e2e-test/tools/run_bf_switchd.py | 56 ++++++ e2e-test/tools/run_bfshell.py | 33 ++++ e2e-test/tools/template/bf_switchd_conf.json | 2 +- e2e-test/tools/test.py | 58 ++++++ e2e-test/tools/util.py | 42 +++- tools/docker_build.sh | 5 + tools/docker_exec_bash.sh | 5 + tools/docker_run_bash.sh | 11 ++ tools/docker_run_smoke_test.sh | 12 ++ tools/setup/install_dep.py | 2 +- 25 files changed, 456 insertions(+), 260 deletions(-) delete mode 100755 docker-build.sh delete mode 100755 docker-join.sh delete mode 100755 docker-run.sh create mode 100644 e2e-test/suites/pna/basicfwd/README.md rename e2e-test/suites/pna/{simple_l2_forwarding/bfshell_cmds.py => basicfwd/cmds_bfshell.py} (53%) create mode 100755 e2e-test/suites/pna/basicfwd/cmds_shell.sh rename e2e-test/suites/pna/{simple_l2_forwarding => basicfwd}/main.p4 (65%) create mode 100755 e2e-test/suites/pna/basicfwd/test.py create mode 100755 e2e-test/tools/clean.py delete mode 100755 e2e-test/tools/cmd.py create mode 100755 e2e-test/tools/compile.py create mode 100644 e2e-test/tools/config.py create mode 100755 e2e-test/tools/run_bf_switchd.py create mode 100755 e2e-test/tools/run_bfshell.py create mode 100755 e2e-test/tools/test.py create mode 100755 tools/docker_build.sh create mode 100755 tools/docker_exec_bash.sh create mode 100755 tools/docker_run_bash.sh create mode 100755 tools/docker_run_smoke_test.sh diff --git a/.dockerignore b/.dockerignore index bb5a071..e7dff1f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,5 @@ docker* e2e-test examples +tools +!tools/setup diff --git a/Dockerfile b/Dockerfile index c96050e..96c40ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ ENV SDE_INSTALL=${SDE}/install ENV LD_LIBRARY_PATH=${SDE_INSTALL}/lib:${SDE_INSTALL}/lib64:${SDE_INSTALL}/lib/x86_64-linux-gnu/ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64 ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib -ENV PATH=${SDE_INSTALL}/bin:${PATH} # Copy repo files in ARG REPO_NAME=p4-dpdk-target diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index 8019dab..0000000 --- a/docker-build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -DOCKER_IMAGE=p4-dpdk-target - -docker build -t $DOCKER_IMAGE . diff --git a/docker-join.sh b/docker-join.sh deleted file mode 100755 index d0429df..0000000 --- a/docker-join.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -DOCKER_CONTAINER=p4-dpdk-target - -docker exec -it $DOCKER_CONTAINER bash diff --git a/docker-run.sh b/docker-run.sh deleted file mode 100755 index e9620c9..0000000 --- a/docker-run.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -DOCKER_IMAGE=p4-dpdk-target -DOCKER_CONTAINER=$DOCKER_IMAGE -DOCKER_MOUNTDIR=/home/work - -docker run -it --rm --privileged \ - --name $DOCKER_CONTAINER \ - -v /dev/hugepages:/dev/hugepages \ - -v "$PWD":$DOCKER_MOUNTDIR -w $DOCKER_MOUNTDIR \ - $DOCKER_IMAGE bash diff --git a/e2e-test/README.md b/e2e-test/README.md index dc25cb8..a778934 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -15,8 +15,10 @@ suites/ P4 compiler outputs generated from main.p4. (generated) bf_switchd_config.json: Config file needed by the bf_switchd binary. - bfshell_cmds.py: + cmds_bfshell.py: Commands to run with bfshell. + cmds_shell.sh: + Commands to run with normal shell. main.p4: The P4 pipeline code. @@ -24,22 +26,66 @@ tools/ Tools used to run the tests. ``` -## Example testing workflow +## Testing environement + +You can either build P4 DPDK target locally or build a Docker image. + +To build the Docker image, at the root of this repo, do: + +```console +./tools/docker_build.sh +``` + +Then you can run `bash` in this container: + +```console +./tools/docker_run_bash.sh +``` + +Or run another `bash` in the same running container: + +```console +./tools/docker_exec_bash.sh +``` + +The last command is useful to provide multiple terminals in the same container. + +## Simple automated smoke test + +To run the smoke test in a Docker container, at the **root of this repo**, do: + +```console +./tools/docker_run_smoke_test.sh +``` + +To run it locally, at **this directory**, do: + +```console +./tools/test.py suites/pna/basicfwd +``` + +## Manual testing workflow + +In the previous section, all testing steps are executed with a single script. If for some reason (e.g. debugging) you would like to execute these steps separately, the details are explained in this section. + +Note that all commands in this section are executed from **this directory**. ### Compile P4 code ```console -./tools/cmd.py compile suites/pna/simple_l2_forwarding +./tools/compile.py suites/pna/basicfwd ``` The following paths will be generated: -- `suites/pna/simple_l2_forwarding/build/` +- `suites/pna/basicfwd/p4c_gen/` ### Run `bf_switchd` +In terminal 1, do: + ```console -./tools/cmd.py bf_switchd suites/pna/simple_l2_forwarding +./tools/run_bf_switchd.py suites/pna/basicfwd ``` The following paths will be generated: @@ -49,14 +95,27 @@ The following paths will be generated: ### Run `bfshell` -Keep the `bf_switchd` terminal open. In another terminal, run: +In terminal 2, do: + +```console +./tools/run_bfshell.py suites/pna/basicfwd +``` + +The following paths will be generated: + +- `suites/pna/simple_l2_forwarding/log/bfshell/` + +### Run testing script + +In terminal 3, do: ```console -./tools/cmd.py bfshell suites/pna/simple_l2_forwarding +./suites/pna/basicfwd/cmds_shell.sh +./suites/pna/basicfwd/test.py ``` ### Clean up (optional) ```console -./tools/cmd.py clean suites/pna/simple_l2_forwarding +./tools/clean.py suites/pna/basicfwd ``` diff --git a/e2e-test/suites/pna/basicfwd/README.md b/e2e-test/suites/pna/basicfwd/README.md new file mode 100644 index 0000000..e63eb46 --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/README.md @@ -0,0 +1,3 @@ +# Basic Forwarding Test + +Receive packets on a port and forward them on the paired port. The mapping is 0 -> 1, 1 -> 0, 2 -> 3, 3 -> 2, etc. diff --git a/e2e-test/suites/pna/simple_l2_forwarding/bfshell_cmds.py b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py similarity index 53% rename from e2e-test/suites/pna/simple_l2_forwarding/bfshell_cmds.py rename to e2e-test/suites/pna/basicfwd/cmds_bfshell.py index 6ff369a..471dfc8 100644 --- a/e2e-test/suites/pna/simple_l2_forwarding/bfshell_cmds.py +++ b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py @@ -16,19 +16,7 @@ MTU=1500 ) -# Add entries - -from netaddr import IPAddress - -control = tdi.main.pipe.MainControl -table = control.forwarding -table.add_with_mark_and_forward( - dst_addr=IPAddress('192.168.1.101'), - marker=0x01, - port=1 -) -table.add_with_mark_and_forward( - dst_addr=IPAddress('192.168.1.102'), - marker=0x02, - port=2 -) +# TODO: Replace the following hack with something better. +foo = "fini" +bar = "shed" +print(foo + bar) diff --git a/e2e-test/suites/pna/basicfwd/cmds_shell.sh b/e2e-test/suites/pna/basicfwd/cmds_shell.sh new file mode 100755 index 0000000..4c230a0 --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/cmds_shell.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Set up TAP ports +for port in {0..3}; do + ip link set "TAP$port" up +done + +# Disable IPv6 because it can interfere with packet sniffing +sysctl -w net.ipv6.conf.all.disable_ipv6=1 +sysctl -w net.ipv6.conf.default.disable_ipv6=1 +sysctl -w net.ipv6.conf.lo.disable_ipv6=1 diff --git a/e2e-test/suites/pna/simple_l2_forwarding/main.p4 b/e2e-test/suites/pna/basicfwd/main.p4 similarity index 65% rename from e2e-test/suites/pna/simple_l2_forwarding/main.p4 rename to e2e-test/suites/pna/basicfwd/main.p4 index e414165..a0c18ff 100644 --- a/e2e-test/suites/pna/simple_l2_forwarding/main.p4 +++ b/e2e-test/suites/pna/basicfwd/main.p4 @@ -7,13 +7,8 @@ header eth_h { bit<16> ether_type; } -header custom_h { - bit<32> marker; -} - struct header_t { eth_h eth; - custom_h custom; } struct metadata_t {} @@ -26,7 +21,6 @@ parser MainParser( ) { state start { pkt.extract(hdr.eth); - pkt.extract(hdr.custom); transition accept; } } @@ -46,31 +40,13 @@ control MainControl( in pna_main_input_metadata_t istd, inout pna_main_output_metadata_t ostd ) { - action mark_and_forward( - bit<32> marker, - PortId_t port - ) { - hdr.custom.marker = marker; - send_to_port(port); - } - - action drop() { - drop_packet(); - } - - table forwarding { - key = { - hdr.eth.dst_addr: exact; - } - actions = { - mark_and_forward; - drop; - } - const default_action = drop(); - } - apply { - forwarding.apply(); + switch (istd.input_port) { + (PortId_t) 0: { send_to_port((PortId_t) 1); } + (PortId_t) 1: { send_to_port((PortId_t) 0); } + (PortId_t) 2: { send_to_port((PortId_t) 3); } + (PortId_t) 3: { send_to_port((PortId_t) 2); } + } } } diff --git a/e2e-test/suites/pna/basicfwd/test.py b/e2e-test/suites/pna/basicfwd/test.py new file mode 100755 index 0000000..1da1564 --- /dev/null +++ b/e2e-test/suites/pna/basicfwd/test.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import time +import unittest + +from scapy.all import IP, UDP, AsyncSniffer, Ether, sendp + + +class ScapyTestCase(unittest.TestCase): + WAIT_TIME = 2 + + # TODO: Make the following test pass. + # def test_one_packet_received_at_expected_port(self): + # sniffer = AsyncSniffer(iface="TAP1", count=8) + # sniffer.start() + + # pkt = Ether() / IP() / UDP() / "Hello, World!" + # sendp(pkt, iface="TAP0") + # pkt_sent = pkt + + # time.sleep(self.WAIT_TIME) + # pkts_recvd = sniffer.stop() + + # self.assertEqual(len(pkts_recvd), 1) + # pkt_recvd = pkts_recvd[0] + # self.assertEqual(pkt_recvd, pkt_sent) + + def test_no_packet_received_at_other_ports(self): + sniffer = AsyncSniffer(iface=["TAP2", "TAP3"], count=8) + sniffer.start() + + pkt = Ether() / IP() / UDP() / "Hello, World!" + sendp(pkt, iface="TAP0") + + time.sleep(self.WAIT_TIME) + pkts_recvd = sniffer.stop() + + self.assertEqual(len(pkts_recvd), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tools/clean.py b/e2e-test/tools/clean.py new file mode 100755 index 0000000..0185ccf --- /dev/null +++ b/e2e-test/tools/clean.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import argparse +import logging +from pathlib import Path + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + clean(args.test_dir) + + +def clean(test_dir): + util.assert_exists(test_dir) + + for path in config.SUITE_PATH_GENERATED: + util.remove_file_or_dir(test_dir / path) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/cmd.py b/e2e-test/tools/cmd.py deleted file mode 100755 index 766aa73..0000000 --- a/e2e-test/tools/cmd.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import logging -import string -from pathlib import Path -from subprocess import run - -import util - -THIS_DIR = Path(__file__).resolve().parent - -# Test suite path conventions - -SUITE_MAIN_P4 = "main.p4" -SUITE_BF_SWITCHD_CONF = "bf_switchd_conf.json" -SUITE_BFSHELL_CMDS = "bfshell_cmds.py" - -SUITE_P4C_GEN = "p4c_gen" -SUITE_P4INFO = SUITE_P4C_GEN + "/p4info.txt" -SUITE_BFRT = SUITE_P4C_GEN + "/bfrt.json" -SUITE_TDI = SUITE_P4C_GEN + "/tdi.json" -SUITE_CONTEXT = SUITE_P4C_GEN + "/context.json" -SUITE_SPEC = SUITE_P4C_GEN + "/config.spec" - -SUITE_LOG = "log" - -SUITE_PATH_GENERATED = [SUITE_BF_SWITCHD_CONF, SUITE_P4C_GEN, SUITE_LOG] - - -# Main entry point -def main(): - logging.basicConfig(level=logging.INFO) - - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(required=True) - - add_parser_for_compile(subparsers) - add_parser_for_bf_switchd(subparsers) - add_parser_for_bfshell(subparsers) - add_parser_for_clean(subparsers) - - args = parser.parse_args() - args.func(args) - - -# The compile command - - -def add_parser_for_compile(subparsers): - parser = subparsers.add_parser("compile") - parser.add_argument("test_dir", type=Path) - parser.set_defaults(func=compile) - - -def compile(args): - test_dir = args.test_dir - - # Make sure test dir exists - assert test_dir.exists(), f"{test_dir} doesn't exist" - - # Prepare compiler output dir - output_dir = test_dir / SUITE_P4C_GEN - if output_dir.exists(): - logging.info(f"Removing existing compiler output dir {output_dir}") - util.remove_file_or_dir(output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - # Compile P4 source - arch = test_dir.parent.stem - test_dir_prefix = test_dir.as_posix() + "/" - p4_source = test_dir_prefix + SUITE_MAIN_P4 - cmd = ["p4c-dpdk"] - cmd += ["--arch", arch] - cmd += ["--p4runtime-files", test_dir_prefix + SUITE_P4INFO] - cmd += ["--bf-rt-schema", test_dir_prefix + SUITE_BFRT] - cmd += ["--tdi", test_dir_prefix + SUITE_TDI] - cmd += ["--context", test_dir_prefix + SUITE_CONTEXT] - cmd += ["-o", test_dir_prefix + SUITE_SPEC] - cmd += [p4_source] - logging.info(f"Compiling {p4_source}") - util.log_cmd(cmd) - run(cmd, check=True) - logging.info("Compiling succeeds") - - -# The bf_switchd command - - -def add_parser_for_bf_switchd(subparsers): - parser = subparsers.add_parser("bf_switchd") - parser.add_argument("test_dir", type=Path) - parser.set_defaults(func=bf_switchd) - - -def bf_switchd(args): - test_dir = args.test_dir - - # Make sure test dir exists - assert test_dir.exists(), f"{test_dir} doesn't exist" - - # Generate bf_switchd conf file - conf_template_path = THIS_DIR / "template" / SUITE_BF_SWITCHD_CONF - conf_template = string.Template(conf_template_path.read_text()) - # These paths have to be absolute, - # or they will be interpreted as relative to $SDE_INSTALL - test_dir_prefix = test_dir.resolve().as_posix() + "/" - conf = conf_template.substitute( - bfrt_config=test_dir_prefix + SUITE_BFRT, - context=test_dir_prefix + SUITE_CONTEXT, - config=test_dir_prefix + SUITE_SPEC, - ) - conf_path = test_dir / SUITE_BF_SWITCHD_CONF - conf_path.write_text(conf) - - # Configure hugepages for DPDK - run("dpdk-hugepages.py -p 1G --setup 2G", shell=True, check=True) - run("dpdk-hugepages.py -s", shell=True, check=True) - - # Run bf_switchd - # Have to switch to the SDE environment. - sde_env = util.get_sde_env() - sde_install = sde_env["SDE_INSTALL"] - bin = f"{sde_install}/bin/bf_switchd" - assert Path(bin).exists(), f"Required binary {bin} doesn't exist" - log_dir = test_dir / "log/bf_switchd/" - log_dir.mkdir(parents=True, exist_ok=True) - cmd = [bin] - cmd += ["--install-dir", sde_env["SDE_INSTALL"]] - cmd += ["--conf-file", conf_path.resolve().as_posix()] - util.log_cmd(cmd) - run(cmd, cwd=log_dir, env=sde_env, check=True) - - -# The bfshell command - - -def add_parser_for_bfshell(subparsers): - parser = subparsers.add_parser("bfshell") - parser.add_argument("test_dir", type=Path) - parser.set_defaults(func=bfshell) - - -def bfshell(args): - test_dir = args.test_dir - - # Make sure test dir exists - assert test_dir.exists(), f"{test_dir} doesn't exist" - - # Run bfshell - # Have to switch to the SDE environment. - sde_env = util.get_sde_env() - sde_install = sde_env["SDE_INSTALL"] - bin = f"{sde_install}/bin/bfshell" - assert Path(bin).exists(), f"Required binary {bin} doesn't exist" - cmd = [bin] - cmd += ["-f", (test_dir / SUITE_BFSHELL_CMDS).resolve().as_posix()] - run(cmd, env=sde_env, check=True) - - -# The clean command - - -def add_parser_for_clean(subparsers): - parser = subparsers.add_parser("clean") - parser.add_argument("test_dir") - parser.set_defaults(func=clean) - - -def clean(args): - test_dir = Path(args.test_dir) - - for path in SUITE_PATH_GENERATED: - util.remove_file_or_dir(test_dir / path) - - -# Start - -if __name__ == "__main__": - main() diff --git a/e2e-test/tools/compile.py b/e2e-test/tools/compile.py new file mode 100755 index 0000000..bee2c13 --- /dev/null +++ b/e2e-test/tools/compile.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import argparse +import logging +from pathlib import Path +from subprocess import run + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + compile(args.test_dir) + + +def compile(test_dir): + util.assert_exists(test_dir) + + # Prepare compiler output dir + output_dir = test_dir / config.SUITE_P4C_GEN + if output_dir.exists(): + logging.info(f"Removing existing compiler output dir {output_dir}") + util.remove_file_or_dir(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Compile P4 source + arch = test_dir.parent.stem + p4_source = test_dir / config.SUITE_MAIN_P4 + cmd = ["p4c-dpdk"] + cmd += ["--arch", arch] + cmd += ["--p4runtime-files", test_dir / config.SUITE_P4INFO] + cmd += ["--bf-rt-schema", test_dir / config.SUITE_BFRT] + cmd += ["--tdi", test_dir / config.SUITE_TDI] + cmd += ["--context", test_dir / config.SUITE_CONTEXT] + cmd += ["-o", test_dir / config.SUITE_SPEC] + cmd += [p4_source] + util.log_cmd(cmd) + run(cmd, check=True) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/config.py b/e2e-test/tools/config.py new file mode 100644 index 0000000..1aa1f44 --- /dev/null +++ b/e2e-test/tools/config.py @@ -0,0 +1,22 @@ +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent + +# Test suite path conventions + +SUITE_MAIN_P4 = "main.p4" +SUITE_BF_SWITCHD_CONF = "bf_switchd_conf.json" +SUITE_CMDS_BFSHELL = "cmds_bfshell.py" +SUITE_CMDS_SHELL = "cmds_shell.sh" +SUITE_TEST = "test.py" + +SUITE_P4C_GEN = "p4c_gen" +SUITE_P4INFO = SUITE_P4C_GEN + "/p4info.txt" +SUITE_BFRT = SUITE_P4C_GEN + "/bfrt.json" +SUITE_TDI = SUITE_P4C_GEN + "/tdi.json" +SUITE_CONTEXT = SUITE_P4C_GEN + "/context.json" +SUITE_SPEC = SUITE_P4C_GEN + "/config.spec" + +SUITE_LOG = "log" + +SUITE_PATH_GENERATED = [SUITE_BF_SWITCHD_CONF, SUITE_P4C_GEN, SUITE_LOG] diff --git a/e2e-test/tools/run_bf_switchd.py b/e2e-test/tools/run_bf_switchd.py new file mode 100755 index 0000000..72cc529 --- /dev/null +++ b/e2e-test/tools/run_bf_switchd.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import string +from pathlib import Path +from subprocess import run + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + run_bf_switchd(args.test_dir) + + +def run_bf_switchd(test_dir, in_background=False): + util.assert_exists(test_dir) + + # Generate bf_switchd conf file + conf_template_path = config.THIS_DIR / "template" / config.SUITE_BF_SWITCHD_CONF + conf_template = string.Template(conf_template_path.read_text()) + # These paths have to be absolute, + # or they will be interpreted as relative to $SDE_INSTALL. + test_dir = test_dir.resolve() + conf = conf_template.substitute( + bfrt_config=test_dir / config.SUITE_BFRT, + context=test_dir / config.SUITE_CONTEXT, + config=test_dir / config.SUITE_SPEC, + ) + conf_path = test_dir / config.SUITE_BF_SWITCHD_CONF + conf_path.write_text(conf) + + # Configure hugepages for DPDK + sde_env = util.get_sde_env() + sde_install = sde_env["SDE_INSTALL"] + run(f"{sde_install}/bin/dpdk-hugepages.py -p 2M --setup 32M", shell=True) + run(f"{sde_install}/bin/dpdk-hugepages.py -s", shell=True) + + # Run the command + bin_name = "bf_switchd" + bin_path, log_dir, sde_env = util.bf_prepare(test_dir, bin_name) + cmd = [bin_path] + cmd += ["--install-dir", sde_env["SDE_INSTALL"]] + cmd += ["--conf-file", conf_path.resolve()] + return util.bf_run(cmd, log_dir, sde_env, in_background) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/run_bfshell.py b/e2e-test/tools/run_bfshell.py new file mode 100755 index 0000000..81b29d0 --- /dev/null +++ b/e2e-test/tools/run_bfshell.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import argparse +import logging +from pathlib import Path + +import config +import util + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + run_bfshell(args.test_dir) + + +def run_bfshell(test_dir, in_background=False): + util.assert_exists(test_dir) + + # Run the command + bin_name = "bfshell" + bin_path, log_dir, sde_env = util.bf_prepare(test_dir, bin_name) + cmd = [bin_path] + cmd += ["-f", (test_dir / config.SUITE_CMDS_BFSHELL).resolve()] + return util.bf_run(cmd, log_dir, sde_env, in_background) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/template/bf_switchd_conf.json b/e2e-test/tools/template/bf_switchd_conf.json index 677027d..8c371ed 100644 --- a/e2e-test/tools/template/bf_switchd_conf.json +++ b/e2e-test/tools/template/bf_switchd_conf.json @@ -10,7 +10,7 @@ "p4_devices": [ { "device-id": 0, - "eal-args": "dummy -n 4 -c 7", + "eal-args": "dummy -n 4 -c 3", "mempools": [ { "name": "MEMPOOL0", diff --git a/e2e-test/tools/test.py b/e2e-test/tools/test.py new file mode 100755 index 0000000..0ff753b --- /dev/null +++ b/e2e-test/tools/test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import argparse +import atexit +import logging +import time +from pathlib import Path +from subprocess import run + +import config +import util +from clean import clean +from compile import compile +from run_bf_switchd import run_bf_switchd +from run_bfshell import run_bfshell + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument("test_dir", type=Path) + args = parser.parse_args() + + test(args.test_dir) + + +def test(test_dir): + util.assert_exists(test_dir) + + clean(test_dir) + compile(test_dir) + + # Run bf_switchd in background + # Wait until bfshell is up + p, stdout = run_bf_switchd(test_dir, in_background=True) + atexit.register(p.kill) + while not stdout.read_text().endswith("bfshell> "): + time.sleep(1) + + # Run some bfshell commands + # Wait until all commands are finished + p, stdout = run_bfshell(test_dir, in_background=True) + atexit.register(p.kill) + while "finished" not in stdout.read_text(): + time.sleep(1) + + # Run some shell commands + cmd = ["bash", test_dir / config.SUITE_CMDS_SHELL] + run(cmd, check=True) + + # Run the testing script + cmd = ["python3", test_dir / config.SUITE_TEST] + run(cmd) + + +if __name__ == "__main__": + main() diff --git a/e2e-test/tools/util.py b/e2e-test/tools/util.py index c55e9f7..d9711a2 100644 --- a/e2e-test/tools/util.py +++ b/e2e-test/tools/util.py @@ -2,6 +2,43 @@ import os import shutil from pathlib import Path +from subprocess import Popen, run + + +def assert_exists(path): + path = Path(path) + assert path.exists(), f"{path} doesn't exist" + + +def bf_prepare(test_dir, bin_name): + sde_env = get_sde_env() + sde_install = sde_env["SDE_INSTALL"] + bin_path = Path(f"{sde_install}/bin/{bin_name}") + assert bin_path.exists(), f"Required binary {bin_path} doesn't exist" + log_dir = test_dir / f"log/{bin_name}/" + log_dir.mkdir(parents=True, exist_ok=True) + return bin_path, log_dir, sde_env + + +def bf_run(cmd, log_dir, sde_env, in_background): + log_cmd(cmd) + stdout = None + if in_background: + stdout = log_dir / "stdout.log" + stderr = log_dir / "stderr.log" + # Set bufsize to 0 because we need to check stdout in real time later + p = Popen( + cmd, + bufsize=0, + stdout=stdout.open("w"), + stderr=stderr.open("w"), + cwd=log_dir, + env=sde_env, + ) + else: + p = run(cmd, cwd=log_dir, env=sde_env, check=True) + logging.info(f"PID: {p.pid}") + return p, stdout def get_sde_env(): @@ -9,7 +46,7 @@ def get_sde_env(): sde_env.update(os.environ) for env_var in ["SDE", "SDE_INSTALL"]: assert env_var in sde_env, f"Environment variable {env_var} not set" - print(f"Using {env_var}: {sde_env[env_var]}") + logging.info(f"Using {env_var}: {sde_env[env_var]}") sde_install = sde_env["SDE_INSTALL"] sde_env["LD_LIBRARY_PATH"] = f"{sde_install}/lib" sde_env["LD_LIBRARY_PATH"] += f":{sde_install}/lib64" @@ -23,8 +60,9 @@ def get_sde_env(): def log_cmd(cmd): if isinstance(cmd, list): + cmd = [arg.as_posix() if isinstance(arg, Path) else arg for arg in cmd] cmd = " ".join(cmd) - logging.info(f"Run command: {cmd}") + logging.info(f"Run command:\n{cmd}") def remove_file_or_dir(path): diff --git a/tools/docker_build.sh b/tools/docker_build.sh new file mode 100755 index 0000000..8477dc8 --- /dev/null +++ b/tools/docker_build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +IMAGE_NAME=p4lang/p4-dpdk-target + +docker build -t $IMAGE_NAME . diff --git a/tools/docker_exec_bash.sh b/tools/docker_exec_bash.sh new file mode 100755 index 0000000..4de2c93 --- /dev/null +++ b/tools/docker_exec_bash.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +CONTAINER_NAME=p4-dpdk-target + +docker exec -it $CONTAINER_NAME bash diff --git a/tools/docker_run_bash.sh b/tools/docker_run_bash.sh new file mode 100755 index 0000000..1150dda --- /dev/null +++ b/tools/docker_run_bash.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +IMAGE_NAME=p4lang/p4-dpdk-target +CONTAINER_NAME=p4-dpdk-target +MOUNT_DIR=/home/work + +docker run -it --rm --privileged \ + --name $CONTAINER_NAME \ + -v /dev/hugepages:/dev/hugepages \ + -v "$PWD":$MOUNT_DIR -w $MOUNT_DIR \ + $IMAGE_NAME bash diff --git a/tools/docker_run_smoke_test.sh b/tools/docker_run_smoke_test.sh new file mode 100755 index 0000000..6a851c3 --- /dev/null +++ b/tools/docker_run_smoke_test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +IMAGE_NAME=p4lang/p4-dpdk-target +CONTAINER_NAME=p4-dpdk-target +MOUNT_DIR=/home/work + +docker run -it --rm --privileged \ + --name $CONTAINER_NAME \ + -v /dev/hugepages:/dev/hugepages \ + -v "$PWD":$MOUNT_DIR -w $MOUNT_DIR \ + $IMAGE_NAME \ + python3 e2e-test/tools/test.py e2e-test/suites/pna/basicfwd diff --git a/tools/setup/install_dep.py b/tools/setup/install_dep.py index cd772a2..bf09294 100644 --- a/tools/setup/install_dep.py +++ b/tools/setup/install_dep.py @@ -140,7 +140,7 @@ def execute_system_command(command: List[str], ## setup proxies for pip to run for item in pip_packages: - pip3_install_command = ["pip3", "install", item] + pip3_install_command = ["pip3", "install", "--upgrade", item] #pip_install_command = ["pip", "install", item] #pip2_install_command = ["pip2", "install", item] print (execute_system_command (pip3_install_command)[0]) From 85a4ab2a4c9a0e4b6d427d124279a49f4a2bc826 Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Thu, 8 Jun 2023 01:44:47 +0000 Subject: [PATCH 07/14] Switch back to original versions of DPDK and TDI in this repo This brings back the enable() method that seems to be a crucial step in setting up the pipeline. But this also results in some errors. Will try to fix them. --- e2e-test/suites/pna/basicfwd/cmds_bfshell.py | 4 +++- e2e-test/suites/pna/basicfwd/cmds_shell.sh | 8 +++---- e2e-test/suites/pna/basicfwd/test.py | 22 ++++++++++---------- e2e-test/tools/run_bf_switchd.py | 2 +- e2e-test/tools/util.py | 12 ++++++++++- src/lld/dpdk/dpdk_src | 2 +- third-party/tdi | 2 +- 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/e2e-test/suites/pna/basicfwd/cmds_bfshell.py b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py index 471dfc8..39e7e72 100644 --- a/e2e-test/suites/pna/basicfwd/cmds_bfshell.py +++ b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py @@ -1,7 +1,6 @@ tdi_python # Add ports - for port_id in range(4): tdi.port.port.add( DEV_PORT=port_id, @@ -16,6 +15,9 @@ MTU=1500 ) +# Enable TDI program +tdi.main.enable() + # TODO: Replace the following hack with something better. foo = "fini" bar = "shed" diff --git a/e2e-test/suites/pna/basicfwd/cmds_shell.sh b/e2e-test/suites/pna/basicfwd/cmds_shell.sh index 4c230a0..6f72c7e 100755 --- a/e2e-test/suites/pna/basicfwd/cmds_shell.sh +++ b/e2e-test/suites/pna/basicfwd/cmds_shell.sh @@ -2,10 +2,10 @@ # Set up TAP ports for port in {0..3}; do - ip link set "TAP$port" up + sudo ip link set "TAP$port" up done # Disable IPv6 because it can interfere with packet sniffing -sysctl -w net.ipv6.conf.all.disable_ipv6=1 -sysctl -w net.ipv6.conf.default.disable_ipv6=1 -sysctl -w net.ipv6.conf.lo.disable_ipv6=1 +sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 +sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 +sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1 diff --git a/e2e-test/suites/pna/basicfwd/test.py b/e2e-test/suites/pna/basicfwd/test.py index 1da1564..c99fc0f 100755 --- a/e2e-test/suites/pna/basicfwd/test.py +++ b/e2e-test/suites/pna/basicfwd/test.py @@ -10,20 +10,20 @@ class ScapyTestCase(unittest.TestCase): WAIT_TIME = 2 # TODO: Make the following test pass. - # def test_one_packet_received_at_expected_port(self): - # sniffer = AsyncSniffer(iface="TAP1", count=8) - # sniffer.start() + def test_one_packet_received_at_expected_port(self): + sniffer = AsyncSniffer(iface="TAP1", count=8) + sniffer.start() - # pkt = Ether() / IP() / UDP() / "Hello, World!" - # sendp(pkt, iface="TAP0") - # pkt_sent = pkt + pkt = Ether() / IP() / UDP() / "Hello, World!" + sendp(pkt, iface="TAP0") + pkt_sent = pkt - # time.sleep(self.WAIT_TIME) - # pkts_recvd = sniffer.stop() + time.sleep(self.WAIT_TIME) + pkts_recvd = sniffer.stop() - # self.assertEqual(len(pkts_recvd), 1) - # pkt_recvd = pkts_recvd[0] - # self.assertEqual(pkt_recvd, pkt_sent) + self.assertEqual(len(pkts_recvd), 1) + pkt_recvd = pkts_recvd[0] + self.assertEqual(pkt_recvd, pkt_sent) def test_no_packet_received_at_other_ports(self): sniffer = AsyncSniffer(iface=["TAP2", "TAP3"], count=8) diff --git a/e2e-test/tools/run_bf_switchd.py b/e2e-test/tools/run_bf_switchd.py index 72cc529..233b2d7 100755 --- a/e2e-test/tools/run_bf_switchd.py +++ b/e2e-test/tools/run_bf_switchd.py @@ -49,7 +49,7 @@ def run_bf_switchd(test_dir, in_background=False): cmd = [bin_path] cmd += ["--install-dir", sde_env["SDE_INSTALL"]] cmd += ["--conf-file", conf_path.resolve()] - return util.bf_run(cmd, log_dir, sde_env, in_background) + return util.bf_run(cmd, log_dir, sde_env, in_background, with_sudo=True) if __name__ == "__main__": diff --git a/e2e-test/tools/util.py b/e2e-test/tools/util.py index d9711a2..b34a0e3 100644 --- a/e2e-test/tools/util.py +++ b/e2e-test/tools/util.py @@ -20,7 +20,17 @@ def bf_prepare(test_dir, bin_name): return bin_path, log_dir, sde_env -def bf_run(cmd, log_dir, sde_env, in_background): +def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): + if with_sudo: + # Passing environment variables this way is necessary. See + # https://github.com/Yi-Tseng/p4-dpdk-target-notes#start-the-switch + sudo = [ + "sudo", + "-E", + f"PATH={sde_env['PATH']}", + f"LD_LIBRARY_PATH={sde_env['LD_LIBRARY_PATH']}", + ] + cmd = sudo + cmd log_cmd(cmd) stdout = None if in_background: diff --git a/src/lld/dpdk/dpdk_src b/src/lld/dpdk/dpdk_src index 189c02f..baf13c3 160000 --- a/src/lld/dpdk/dpdk_src +++ b/src/lld/dpdk/dpdk_src @@ -1 +1 @@ -Subproject commit 189c02f5ec891ed02927062e124e8ca03adf74e7 +Subproject commit baf13c3135d0c5998fff7edc23fb89412dc89246 diff --git a/third-party/tdi b/third-party/tdi index fe1308c..2f9ce92 160000 --- a/third-party/tdi +++ b/third-party/tdi @@ -1 +1 @@ -Subproject commit fe1308cae6f4e6e673172d17215494987922bee6 +Subproject commit 2f9ce92cd3e2514847faa03157e715ac7b49dc74 From 7f36e982e38b9afdbf16017f927d624845c3f684 Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Thu, 8 Jun 2023 07:54:04 +0000 Subject: [PATCH 08/14] Make the test case pass This is a temporary solution though. --- e2e-test/suites/pna/basicfwd/test.py | 7 +++---- e2e-test/tools/util.py | 19 ++++++++++--------- src/pipe_mgr/shared/dal/dpdk/dal_init.c | 14 +++++++++----- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/e2e-test/suites/pna/basicfwd/test.py b/e2e-test/suites/pna/basicfwd/test.py index c99fc0f..0bd8bf8 100755 --- a/e2e-test/suites/pna/basicfwd/test.py +++ b/e2e-test/suites/pna/basicfwd/test.py @@ -3,13 +3,12 @@ import time import unittest -from scapy.all import IP, UDP, AsyncSniffer, Ether, sendp +from scapy.all import IP, UDP, AsyncSniffer, Ether, raw, sendp class ScapyTestCase(unittest.TestCase): - WAIT_TIME = 2 + WAIT_TIME = 1 - # TODO: Make the following test pass. def test_one_packet_received_at_expected_port(self): sniffer = AsyncSniffer(iface="TAP1", count=8) sniffer.start() @@ -23,7 +22,7 @@ def test_one_packet_received_at_expected_port(self): self.assertEqual(len(pkts_recvd), 1) pkt_recvd = pkts_recvd[0] - self.assertEqual(pkt_recvd, pkt_sent) + self.assertEqual(raw(pkt_recvd), raw(pkt_sent)) def test_no_packet_received_at_other_ports(self): sniffer = AsyncSniffer(iface=["TAP2", "TAP3"], count=8) diff --git a/e2e-test/tools/util.py b/e2e-test/tools/util.py index b34a0e3..b60831f 100644 --- a/e2e-test/tools/util.py +++ b/e2e-test/tools/util.py @@ -5,6 +5,11 @@ from subprocess import Popen, run +def args_to_str(args): + args = [arg.as_posix() if isinstance(arg, Path) else arg for arg in args] + return " ".join(args) + + def assert_exists(path): path = Path(path) assert path.exists(), f"{path} doesn't exist" @@ -21,15 +26,11 @@ def bf_prepare(test_dir, bin_name): def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): + cmd = args_to_str(cmd) if with_sudo: # Passing environment variables this way is necessary. See # https://github.com/Yi-Tseng/p4-dpdk-target-notes#start-the-switch - sudo = [ - "sudo", - "-E", - f"PATH={sde_env['PATH']}", - f"LD_LIBRARY_PATH={sde_env['LD_LIBRARY_PATH']}", - ] + sudo = "sudo -E PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH " cmd = sudo + cmd log_cmd(cmd) stdout = None @@ -39,6 +40,7 @@ def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): # Set bufsize to 0 because we need to check stdout in real time later p = Popen( cmd, + shell=True, bufsize=0, stdout=stdout.open("w"), stderr=stderr.open("w"), @@ -46,7 +48,7 @@ def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): env=sde_env, ) else: - p = run(cmd, cwd=log_dir, env=sde_env, check=True) + p = run(cmd, shell=True, cwd=log_dir, env=sde_env, check=True) logging.info(f"PID: {p.pid}") return p, stdout @@ -70,8 +72,7 @@ def get_sde_env(): def log_cmd(cmd): if isinstance(cmd, list): - cmd = [arg.as_posix() if isinstance(arg, Path) else arg for arg in cmd] - cmd = " ".join(cmd) + cmd = args_to_str(cmd) logging.info(f"Run command:\n{cmd}") diff --git a/src/pipe_mgr/shared/dal/dpdk/dal_init.c b/src/pipe_mgr/shared/dal/dpdk/dal_init.c index 16fbb49..2643ead 100644 --- a/src/pipe_mgr/shared/dal/dpdk/dal_init.c +++ b/src/pipe_mgr/shared/dal/dpdk/dal_init.c @@ -125,11 +125,15 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, snprintf(buffer, sizeof(buffer), "gcc -shared %s -o %s ", o_filepath, so_filepath); status = system(buffer); - if (status) { - LOG_ERROR("%s line:%d Cannot generate %s file\n", - __func__, __LINE__, so_filepath); - return BF_INTERNAL_ERROR; - } + // TODO: Commenting the following out allows e2e-test to pass, + // but this might not be a good solution in the long term. + // What I observed is that the .so file is generated, + // but somehow the status code is -1. + // if (status) { + // LOG_ERROR("%s line:%d Cannot generate %s file\n", + // __func__, __LINE__, so_filepath); + // return BF_INTERNAL_ERROR; + // } fd = fopen(IOSPEC_FILE_PATH, "r"); if (!fd) { From 97f680cdfbdd7714ee06361236b2a01ecc34e29c Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Thu, 8 Jun 2023 18:21:33 +0000 Subject: [PATCH 09/14] General refinement --- e2e-test/README.md | 34 ++++++++++++------- e2e-test/suites/.gitignore | 2 +- e2e-test/tools/config.py | 4 +-- e2e-test/tools/run_bf_switchd.py | 4 +-- ...switchd_conf.json => conf_bf_switchd.json} | 0 e2e-test/tools/util.py | 23 +++++++------ src/pipe_mgr/shared/dal/dpdk/dal_init.c | 19 ++++++----- tools/docker_build.sh | 2 ++ tools/docker_exec_bash.sh | 2 ++ tools/docker_run_bash.sh | 2 ++ tools/docker_run_smoke_test.sh | 2 ++ 11 files changed, 56 insertions(+), 38 deletions(-) rename e2e-test/tools/template/{bf_switchd_conf.json => conf_bf_switchd.json} (100%) diff --git a/e2e-test/README.md b/e2e-test/README.md index a778934..166c6a5 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -13,14 +13,18 @@ suites/ Log files generated during testing. (generated) p4c_gen/ P4 compiler outputs generated from main.p4. - (generated) bf_switchd_config.json: - Config file needed by the bf_switchd binary. cmds_bfshell.py: - Commands to run with bfshell. + Commands to run with bfshell before testing script. cmds_shell.sh: - Commands to run with normal shell. + Commands to run with normal shell before testing script. + (generated) conf_bf_switchd.json: + Config file needed by the bf_switchd binary. main.p4: The P4 pipeline code. + README.md: + A description of the test. + test.py: + The packet testing script. tools/ Tools used to run the tests. @@ -28,31 +32,35 @@ tools/ ## Testing environement -You can either build P4 DPDK target locally or build a Docker image. +You can either build P4 DPDK target locally following the README at the root of this repo, or build a Docker image. Below we introduce the Docker approach. -To build the Docker image, at the root of this repo, do: +All commands in this section are expected to be executed from the root of this repo. + +### Docker image building ```console ./tools/docker_build.sh ``` -Then you can run `bash` in this container: +### Convenient Docker scripts + +To run a `bash` in a new container: ```console ./tools/docker_run_bash.sh ``` -Or run another `bash` in the same running container: +To run a new `bash` in the same running container: ```console ./tools/docker_exec_bash.sh ``` -The last command is useful to provide multiple terminals in the same container. +The last script is useful for providing multiple terminal shells in the same container. ## Simple automated smoke test -To run the smoke test in a Docker container, at the **root of this repo**, do: +To run the smoke test in a container, at the **root of this repo**, do: ```console ./tools/docker_run_smoke_test.sh @@ -66,9 +74,9 @@ To run it locally, at **this directory**, do: ## Manual testing workflow -In the previous section, all testing steps are executed with a single script. If for some reason (e.g. debugging) you would like to execute these steps separately, the details are explained in this section. +In the previous section, all testing steps are executed with a single script. If for some reason (e.g. debugging) you would like to execute these steps separately, the details are described in this section. -Note that all commands in this section are executed from **this directory**. +All commands in this section are expected to be executed from this directory. ### Compile P4 code @@ -91,7 +99,7 @@ In terminal 1, do: The following paths will be generated: - `suites/pna/simple_l2_forwarding/log/bf_switchd/` -- `suites/pna/simple_l2_forwarding/bf_switchd_conf.json` +- `suites/pna/simple_l2_forwarding/conf_bf_switchd.json` ### Run `bfshell` diff --git a/e2e-test/suites/.gitignore b/e2e-test/suites/.gitignore index 23aec05..f832861 100644 --- a/e2e-test/suites/.gitignore +++ b/e2e-test/suites/.gitignore @@ -1,3 +1,3 @@ log/ p4c_gen/ -bf_switchd_conf.json +conf_bf_switchd.json diff --git a/e2e-test/tools/config.py b/e2e-test/tools/config.py index 1aa1f44..3faa463 100644 --- a/e2e-test/tools/config.py +++ b/e2e-test/tools/config.py @@ -5,9 +5,9 @@ # Test suite path conventions SUITE_MAIN_P4 = "main.p4" -SUITE_BF_SWITCHD_CONF = "bf_switchd_conf.json" SUITE_CMDS_BFSHELL = "cmds_bfshell.py" SUITE_CMDS_SHELL = "cmds_shell.sh" +SUITE_CONF_BF_SWITCHD = "conf_bf_switchd.json" SUITE_TEST = "test.py" SUITE_P4C_GEN = "p4c_gen" @@ -19,4 +19,4 @@ SUITE_LOG = "log" -SUITE_PATH_GENERATED = [SUITE_BF_SWITCHD_CONF, SUITE_P4C_GEN, SUITE_LOG] +SUITE_PATH_GENERATED = [SUITE_CONF_BF_SWITCHD, SUITE_P4C_GEN, SUITE_LOG] diff --git a/e2e-test/tools/run_bf_switchd.py b/e2e-test/tools/run_bf_switchd.py index 233b2d7..0335c03 100755 --- a/e2e-test/tools/run_bf_switchd.py +++ b/e2e-test/tools/run_bf_switchd.py @@ -24,7 +24,7 @@ def run_bf_switchd(test_dir, in_background=False): util.assert_exists(test_dir) # Generate bf_switchd conf file - conf_template_path = config.THIS_DIR / "template" / config.SUITE_BF_SWITCHD_CONF + conf_template_path = config.THIS_DIR / "template" / config.SUITE_CONF_BF_SWITCHD conf_template = string.Template(conf_template_path.read_text()) # These paths have to be absolute, # or they will be interpreted as relative to $SDE_INSTALL. @@ -34,7 +34,7 @@ def run_bf_switchd(test_dir, in_background=False): context=test_dir / config.SUITE_CONTEXT, config=test_dir / config.SUITE_SPEC, ) - conf_path = test_dir / config.SUITE_BF_SWITCHD_CONF + conf_path = test_dir / config.SUITE_CONF_BF_SWITCHD conf_path.write_text(conf) # Configure hugepages for DPDK diff --git a/e2e-test/tools/template/bf_switchd_conf.json b/e2e-test/tools/template/conf_bf_switchd.json similarity index 100% rename from e2e-test/tools/template/bf_switchd_conf.json rename to e2e-test/tools/template/conf_bf_switchd.json diff --git a/e2e-test/tools/util.py b/e2e-test/tools/util.py index b60831f..b273d15 100644 --- a/e2e-test/tools/util.py +++ b/e2e-test/tools/util.py @@ -1,15 +1,11 @@ import logging import os import shutil +from functools import lru_cache from pathlib import Path from subprocess import Popen, run -def args_to_str(args): - args = [arg.as_posix() if isinstance(arg, Path) else arg for arg in args] - return " ".join(args) - - def assert_exists(path): path = Path(path) assert path.exists(), f"{path} doesn't exist" @@ -26,11 +22,15 @@ def bf_prepare(test_dir, bin_name): def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): - cmd = args_to_str(cmd) if with_sudo: # Passing environment variables this way is necessary. See # https://github.com/Yi-Tseng/p4-dpdk-target-notes#start-the-switch - sudo = "sudo -E PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH " + sudo = [ + "sudo", + "-E", + "PATH=" + sde_env["PATH"], + "LD_LIBRARY_PATH=" + sde_env["LD_LIBRARY_PATH"], + ] cmd = sudo + cmd log_cmd(cmd) stdout = None @@ -40,7 +40,6 @@ def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): # Set bufsize to 0 because we need to check stdout in real time later p = Popen( cmd, - shell=True, bufsize=0, stdout=stdout.open("w"), stderr=stderr.open("w"), @@ -48,11 +47,12 @@ def bf_run(cmd, log_dir, sde_env, in_background, with_sudo=False): env=sde_env, ) else: - p = run(cmd, shell=True, cwd=log_dir, env=sde_env, check=True) + p = run(cmd, cwd=log_dir, env=sde_env, check=True) logging.info(f"PID: {p.pid}") return p, stdout +@lru_cache(maxsize=None) def get_sde_env(): sde_env = {} sde_env.update(os.environ) @@ -72,8 +72,9 @@ def get_sde_env(): def log_cmd(cmd): if isinstance(cmd, list): - cmd = args_to_str(cmd) - logging.info(f"Run command:\n{cmd}") + cmd = [arg.as_posix() if isinstance(arg, Path) else arg for arg in cmd] + cmd = " ".join(cmd) + logging.info(f"Run command:\n {cmd}\n") def remove_file_or_dir(path): diff --git a/src/pipe_mgr/shared/dal/dpdk/dal_init.c b/src/pipe_mgr/shared/dal/dpdk/dal_init.c index 2643ead..4aad343 100644 --- a/src/pipe_mgr/shared/dal/dpdk/dal_init.c +++ b/src/pipe_mgr/shared/dal/dpdk/dal_init.c @@ -125,15 +125,16 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, snprintf(buffer, sizeof(buffer), "gcc -shared %s -o %s ", o_filepath, so_filepath); status = system(buffer); - // TODO: Commenting the following out allows e2e-test to pass, - // but this might not be a good solution in the long term. - // What I observed is that the .so file is generated, - // but somehow the status code is -1. - // if (status) { - // LOG_ERROR("%s line:%d Cannot generate %s file\n", - // __func__, __LINE__, so_filepath); - // return BF_INTERNAL_ERROR; - // } + if (status) { + // TODO: Using LOG_WARN instead of LOG_ERROR, and commenting out + // return below is a hack to make e2e-test pass. What I observed + // is that so_filepath is actually generated, but somehow status + // is -1, leading to this error handling branch. Should try to fix + // this issue in a better way. + LOG_WARN("%s line:%d Cannot generate %s file\n", + __func__, __LINE__, so_filepath); + // return BF_INTERNAL_ERROR; + } fd = fopen(IOSPEC_FILE_PATH, "r"); if (!fd) { diff --git a/tools/docker_build.sh b/tools/docker_build.sh index 8477dc8..5b7a027 100755 --- a/tools/docker_build.sh +++ b/tools/docker_build.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# +# Build a Docker image for the P4 DPDK stack. IMAGE_NAME=p4lang/p4-dpdk-target diff --git a/tools/docker_exec_bash.sh b/tools/docker_exec_bash.sh index 4de2c93..b67f4b2 100755 --- a/tools/docker_exec_bash.sh +++ b/tools/docker_exec_bash.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# +# Run a new bash in a running container. CONTAINER_NAME=p4-dpdk-target diff --git a/tools/docker_run_bash.sh b/tools/docker_run_bash.sh index 1150dda..4606362 100755 --- a/tools/docker_run_bash.sh +++ b/tools/docker_run_bash.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# +# Run a bash in a new container. IMAGE_NAME=p4lang/p4-dpdk-target CONTAINER_NAME=p4-dpdk-target diff --git a/tools/docker_run_smoke_test.sh b/tools/docker_run_smoke_test.sh index 6a851c3..0e26ed6 100755 --- a/tools/docker_run_smoke_test.sh +++ b/tools/docker_run_smoke_test.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# +# Run the smoke test in a container. IMAGE_NAME=p4lang/p4-dpdk-target CONTAINER_NAME=p4-dpdk-target From e86daa3d0b62bf7f711ab99fbe420a90d9fc7e14 Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Thu, 8 Jun 2023 19:00:14 +0000 Subject: [PATCH 10/14] Add forwarding table back to the P4 pipeline It works this time! --- e2e-test/suites/pna/basicfwd/cmds_bfshell.py | 8 ++++++ e2e-test/suites/pna/basicfwd/main.p4 | 26 +++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/e2e-test/suites/pna/basicfwd/cmds_bfshell.py b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py index 39e7e72..4a3e240 100644 --- a/e2e-test/suites/pna/basicfwd/cmds_bfshell.py +++ b/e2e-test/suites/pna/basicfwd/cmds_bfshell.py @@ -18,6 +18,14 @@ # Enable TDI program tdi.main.enable() +# Add entries +control = tdi.main.pipe.MainControl +table = control.forwarding +table.add_with_forward(input_port=0, output_port=1) +table.add_with_forward(input_port=1, output_port=0) +table.add_with_forward(input_port=2, output_port=3) +table.add_with_forward(input_port=3, output_port=2) + # TODO: Replace the following hack with something better. foo = "fini" bar = "shed" diff --git a/e2e-test/suites/pna/basicfwd/main.p4 b/e2e-test/suites/pna/basicfwd/main.p4 index a0c18ff..1b0df3f 100644 --- a/e2e-test/suites/pna/basicfwd/main.p4 +++ b/e2e-test/suites/pna/basicfwd/main.p4 @@ -40,13 +40,27 @@ control MainControl( in pna_main_input_metadata_t istd, inout pna_main_output_metadata_t ostd ) { - apply { - switch (istd.input_port) { - (PortId_t) 0: { send_to_port((PortId_t) 1); } - (PortId_t) 1: { send_to_port((PortId_t) 0); } - (PortId_t) 2: { send_to_port((PortId_t) 3); } - (PortId_t) 3: { send_to_port((PortId_t) 2); } + action forward(PortId_t output_port) { + send_to_port(output_port); + } + + action drop() { + drop_packet(); + } + + table forwarding { + key = { + istd.input_port: exact; } + actions = { + forward; + drop; + } + const default_action = drop(); + } + + apply { + forwarding.apply(); } } From f8b9480530840182d66866fce265f8266f9498ed Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Thu, 22 Jun 2023 17:23:32 +0000 Subject: [PATCH 11/14] Log command and returned status --- src/pipe_mgr/shared/dal/dpdk/dal_init.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/pipe_mgr/shared/dal/dpdk/dal_init.c b/src/pipe_mgr/shared/dal/dpdk/dal_init.c index 4aad343..34b3c8a 100644 --- a/src/pipe_mgr/shared/dal/dpdk/dal_init.c +++ b/src/pipe_mgr/shared/dal/dpdk/dal_init.c @@ -114,7 +114,11 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, snprintf(buffer, sizeof(buffer), "gcc -c -O3 -fpic " "-Wno-deprecated-declarations -o %s %s -I %s", o_filepath, c_filepath, i_filepath); + LOG_TRACE("%s line:%d Running command: %s\n", + __func__, __LINE__, buffer); status = system(buffer); + LOG_TRACE("%s line:%d Returned status: %d\n", + __func__, __LINE__, status); if (status) { LOG_ERROR("%s line:%d Cannot generate %s file\n", __func__, __LINE__, o_filepath); @@ -124,16 +128,15 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, memset(buffer, 0, sizeof(buffer)); snprintf(buffer, sizeof(buffer), "gcc -shared %s -o %s ", o_filepath, so_filepath); + LOG_TRACE("%s line:%d Running command: %s\n", + __func__, __LINE__, buffer); status = system(buffer); + LOG_TRACE("%s line:%d Returned status: %d\n", + __func__, __LINE__, status); if (status) { - // TODO: Using LOG_WARN instead of LOG_ERROR, and commenting out - // return below is a hack to make e2e-test pass. What I observed - // is that so_filepath is actually generated, but somehow status - // is -1, leading to this error handling branch. Should try to fix - // this issue in a better way. - LOG_WARN("%s line:%d Cannot generate %s file\n", + LOG_ERROR("%s line:%d Cannot generate %s file\n", __func__, __LINE__, so_filepath); - // return BF_INTERNAL_ERROR; + return BF_INTERNAL_ERROR; } fd = fopen(IOSPEC_FILE_PATH, "r"); From ecdbba41dcbda284b0cdda5908a9a87841e8a949 Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Thu, 22 Jun 2023 20:15:51 +0000 Subject: [PATCH 12/14] Use zlog-cfg file to configure log level instead of hard coding --- bf_switchd/bf_switchd.c | 9 --------- zlog-cfg | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/bf_switchd/bf_switchd.c b/bf_switchd/bf_switchd.c index 985bae4..769c3ee 100644 --- a/bf_switchd/bf_switchd.c +++ b/bf_switchd/bf_switchd.c @@ -1565,15 +1565,6 @@ static int bf_switchd_sys_init(void) { return ret; } - /* For performance set defaut log level to error for pipe and pkt modules */ - bf_sys_log_level_set(BF_MOD_PIPE, BF_LOG_DEST_FILE, BF_LOG_ERR); - bf_sys_trace_level_set(BF_MOD_PIPE, BF_LOG_ERR); - -#ifdef BFRT_ENABLED - bf_sys_log_level_set(BF_MOD_BFRT, BF_LOG_DEST_FILE, BF_LOG_ERR); - bf_sys_trace_level_set(BF_MOD_BFRT, BF_LOG_ERR); -#endif - return 0; } diff --git a/zlog-cfg b/zlog-cfg index c0716d0..04e6fa9 100644 --- a/zlog-cfg +++ b/zlog-cfg @@ -23,7 +23,7 @@ BF_LLD.ERROR >stdout;console_format BF_LLD.DEBUG "p4_driver.log", 5M * 5 ;file_format BF_PIPE.ERROR >stdout;console_format -BF_PIPE.DEBUG "p4_driver.log", 5M * 5 ;file_format +BF_PIPE.ERROR "p4_driver.log", 5M * 5 ;file_format BF_DVM.ERROR >stdout;console_format BF_DVM.DEBUG "p4_driver.log", 5M * 5 ;file_format @@ -41,7 +41,7 @@ BF_PM.ERROR >stdout;console_format BF_PM.DEBUG "p4_driver.log", 5M * 5 ;file_format BF_BFRT.ERROR >stdout;console_format -BF_BFRT.DEBUG "p4_driver.log", 5M * 5 ;file_format +BF_BFRT.ERROR "p4_driver.log", 5M * 5 ;file_format KRNLMON.ERROR >stdout;console_format KRNLMON.DEBUG "krnlmon.log", 5M * 5 ;file_format From b1bb5a5d4b8c0baa86fe9275c703a3110bb9805e Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Thu, 22 Jun 2023 23:14:11 +0000 Subject: [PATCH 13/14] Switch to checking file existence as a temporary workaround --- src/pipe_mgr/shared/dal/dpdk/dal_init.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/pipe_mgr/shared/dal/dpdk/dal_init.c b/src/pipe_mgr/shared/dal/dpdk/dal_init.c index 34b3c8a..35a76a8 100644 --- a/src/pipe_mgr/shared/dal/dpdk/dal_init.c +++ b/src/pipe_mgr/shared/dal/dpdk/dal_init.c @@ -114,11 +114,8 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, snprintf(buffer, sizeof(buffer), "gcc -c -O3 -fpic " "-Wno-deprecated-declarations -o %s %s -I %s", o_filepath, c_filepath, i_filepath); - LOG_TRACE("%s line:%d Running command: %s\n", - __func__, __LINE__, buffer); + LOG_TRACE("Running command: %s\n", buffer); status = system(buffer); - LOG_TRACE("%s line:%d Returned status: %d\n", - __func__, __LINE__, status); if (status) { LOG_ERROR("%s line:%d Cannot generate %s file\n", __func__, __LINE__, o_filepath); @@ -126,18 +123,25 @@ int dal_enable_pipeline(bf_dev_id_t dev_id, } memset(buffer, 0, sizeof(buffer)); - snprintf(buffer, sizeof(buffer), "gcc -shared %s -o %s ", + snprintf(buffer, sizeof(buffer), "gcc -shared %s -o %s", o_filepath, so_filepath); - LOG_TRACE("%s line:%d Running command: %s\n", - __func__, __LINE__, buffer); + LOG_TRACE("Running command: %s\n", buffer); status = system(buffer); - LOG_TRACE("%s line:%d Returned status: %d\n", - __func__, __LINE__, status); if (status) { + LOG_ERROR("%s line:%d Command (%s) failed with exit code: %d\n", + __func__, __LINE__, buffer, status); + } + // TODO: Ideally, we should check whether status is 0 below. But + // sometimes the returned status code is -1, even when the .so file is + // generated properly. Checking file existence is a temporary + // workaround. Need to further investigate and solve this issue. + fd = fopen(so_filepath, "r"); + if (!fd) { LOG_ERROR("%s line:%d Cannot generate %s file\n", __func__, __LINE__, so_filepath); return BF_INTERNAL_ERROR; } + fclose(fd); fd = fopen(IOSPEC_FILE_PATH, "r"); if (!fd) { From 24a215b4dad9006137310a23769caf77ee1afc31 Mon Sep 17 00:00:00 2001 From: Bili Dong Date: Sat, 24 Jun 2023 00:28:50 +0000 Subject: [PATCH 14/14] Use multi-stage build to reduce Docker image size Size is reduced from 4.78GB to 1GB! Thanks @chrispsommers for the suggestion. --- Dockerfile | 95 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96c40ba..b5c6f36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,29 @@ -ARG BASE_IMAGE=p4lang/p4c:latest -FROM ${BASE_IMAGE} -LABEL maintainer="P4 Developers " +# Share args between stages. +# See https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact. +# They should be consistent with tools/setup/p4sde_env_setup.sh. +ARG SDE=/home/sde +ARG SDE_INSTALL=${SDE}/install +ARG LD_LIBRARY_PATH=${SDE_INSTALL}/lib:${SDE_INSTALL}/lib64:${SDE_INSTALL}/lib/x86_64-linux-gnu/ +ARG LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64 +ARG LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib + +#====================================== +# Stage 1/3: Reuse upstream p4c image. +#====================================== +# FROM p4lang/p4c:latest as p4c +# :latest on 2023-06-23: +FROM p4lang/p4c@sha256:c1cbb66cea83de50b43d7ef78d478dd5e43ce9e1116921d6700cc40bb505e12a as p4c -# Install some packages needed for later steps +#============================================== +# Stage 2/3: Build P4 DPDK target from source. +#============================================== +FROM ubuntu:20.04 as p4-dpdk-target ARG DEBIAN_FRONTEND=noninteractive +ARG SDE +ARG SDE_INSTALL +ARG LD_LIBRARY_PATH + +# Install core packages needed for later steps. RUN apt-get update && apt-get install -y \ python-is-python3 \ python3 \ @@ -11,32 +31,73 @@ RUN apt-get update && apt-get install -y \ sudo \ && rm -rf /var/lib/apt/lists/* -# Set environment variables -# We don't simply source tools/setup/p4sde_env_setup.sh, -# because we want to persist these in the running container. -ENV SDE=/home/sde -ENV SDE_INSTALL=${SDE}/install -ENV LD_LIBRARY_PATH=${SDE_INSTALL}/lib:${SDE_INSTALL}/lib64:${SDE_INSTALL}/lib/x86_64-linux-gnu/ -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib64 -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib - -# Copy repo files in +# Copy repo files in. ARG REPO_NAME=p4-dpdk-target COPY . ${SDE}/${REPO_NAME}/ -# Install dependent packages +# Install dependent packages. WORKDIR ${SDE}/${REPO_NAME}/tools/setup RUN pip3 install distro \ && apt-get update \ && python3 install_dep.py \ && rm -rf /var/lib/apt/lists/* -# Build P4 DPDK target +# Build P4 DPDK target. WORKDIR ${SDE}/${REPO_NAME} RUN ./autogen.sh \ && ./configure --prefix=${SDE_INSTALL} \ && make -j \ && make install -# Set working directory +#=================================== +# Stage 3/3: Construct final image. +#=================================== +FROM ubuntu:20.04 +LABEL maintainer="P4 Developers " +ARG DEBIAN_FRONTEND=noninteractive +ARG SDE +ARG SDE_INSTALL +ARG LD_LIBRARY_PATH + +# Persist essential environment variables. +ENV SDE=${SDE} +ENV SDE_INSTALL=${SDE_INSTALL} +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH} + +# Install some useful tools. +RUN apt-get update && apt-get install -y \ + iproute2 \ + python-is-python3 \ + python3 \ + python3-pip \ + sudo \ + && rm -rf /var/lib/apt/lists/* +RUN pip3 install --upgrade \ + scapy + +# [from p4c] +# Just copy minimal p4c bins & libs for dpdk backend. +# Adapted from https://github.com/sonic-net/DASH/blob/main/dash-pipeline/dockerfiles/Dockerfile.p4c-dpdk. +COPY --from=p4c \ + /usr/local/bin/p4c \ + /usr/local/bin/p4c-dpdk \ + /usr/local/bin/ +COPY --from=p4c \ + /usr/lib/x86_64-linux-gnu/libboost_*so* \ + /usr/lib/x86_64-linux-gnu/libgc.so* \ + /usr/lib/x86_64-linux-gnu/ +COPY --from=p4c \ + /usr/local/share/p4c/ \ + /usr/local/share/p4c/ + +# [from p4-dpdk-target] +# Just copy SDE_INSTALL dir & minimal system libs. +COPY --from=p4-dpdk-target \ + ${SDE_INSTALL} \ + ${SDE_INSTALL} +COPY --from=p4-dpdk-target \ + /usr/lib/x86_64-linux-gnu/libedit.so* \ + /usr/lib/x86_64-linux-gnu/ + +# Set default working directory. WORKDIR ${SDE}