From 3320f9fb8a403da96d9eb7742f12032de8012dcc Mon Sep 17 00:00:00 2001 From: rickwu4444 Date: Thu, 30 Nov 2023 09:14:39 +0800 Subject: [PATCH] Add TCP test The TCP echo stress is migrate from project erlangen. --- .../bin/tcp_multi_connections.py | 237 ++++++++++++++++++ .../bin/tcpecho_stress.sh | 179 +++++++++++++ .../units/ethernet/jobs.pxu | 42 ++++ .../units/ethernet/manifest.pxu | 9 + .../units/ethernet/test-plan.pxu | 34 +++ .../units/test-plan-ce-oem.pxu | 3 + 6 files changed, 504 insertions(+) create mode 100755 checkbox-provider-ce-oem/bin/tcp_multi_connections.py create mode 100755 checkbox-provider-ce-oem/bin/tcpecho_stress.sh create mode 100644 checkbox-provider-ce-oem/units/ethernet/jobs.pxu create mode 100644 checkbox-provider-ce-oem/units/ethernet/manifest.pxu create mode 100644 checkbox-provider-ce-oem/units/ethernet/test-plan.pxu diff --git a/checkbox-provider-ce-oem/bin/tcp_multi_connections.py b/checkbox-provider-ce-oem/bin/tcp_multi_connections.py new file mode 100755 index 0000000..2eb9f16 --- /dev/null +++ b/checkbox-provider-ce-oem/bin/tcp_multi_connections.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +import socket +import argparse +import random +import threading +import logging +import time +from datetime import datetime, timedelta + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler(), + ], +) + + +def server(start_port, end_port): + """ + Start the server to listen on a range of ports. + + Args: + - start_port (int): Starting port for the server. + - end_port (int): Ending port for the server. + """ + for port in range(start_port, end_port+1): + threading.Thread(target=handle_port, args=(port,)).start() + + +def handle_port(port): + """ + Handle incoming connections on the specified port. + + Args: + - port (int): Port to handle connections. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_socket: + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(('0.0.0.0', port)) + server_socket.listen() + + logging.info("Server listening on port {}".format(port)) + + while True: + conn, addr = server_socket.accept() + threading.Thread(target=handle_client, args=(conn, addr)).start() + + +def handle_client(conn, addr): + """ + Handle a client connection. + + Args: + - conn (socket): Client socket connection. + - addr (str): Client address. + """ + with conn: + logging.info("Connected by {}.".format(addr)) + received_data = b'' + while True: + data = conn.recv(1024) + if not data: + break + received_data += data + if received_data: + conn.shutdown(socket.SHUT_RD) + logging.info("Received raw data from {}.".format(addr)) + conn.sendall(received_data) + + +def client(host, start_port, end_port, payload, done_event, start_time): + """ + Start the client to connect to a range of server ports. + + Args: + - host (str): Server host. + - start_port (int): Starting port for the client. + - end_port (int): Ending port for the client. + - payload (str): Payload to send to the server. + - done_event (threading.Event): Event to signal when the client is done. + - start_time (datetime): Time until which the client should run. + """ + threads = [] + for port in range(start_port, end_port+1): + thread = threading.Thread(target=send_payload, + args=(host, port, payload, start_time)) + threads.append(thread) + thread.start() + + # Wait for all client threads to finish + for thread in threads: + thread.join() + + done_event.set() + + +def send_payload(host, port, payload, start_time): + """ + Send a payload to the specified port and handle the server response. + + Args: + - host (str): Server host. + - port (int): Port to connect to. + - payload (str): Payload to send to the server. + - start_time (datetime): Time until which the client should run. + """ + try: + with socket.create_connection( + (host, port), timeout=120) as client_socket: + logging.info("Connect to port {}".format(port)) + while datetime.now() < start_time: + time.sleep(1) + logging.info("Sending payload to port {}.".format(port)) + # Sending payload for 30 sec after start sending. + count = 0 + while datetime.now() < start_time + timedelta(seconds=30): + client_socket.sendall(payload.encode()) + ''' + Introduce a random interval between 1 and 3 seconds. + We try to control each port will send 64K payload + at least 10 times in 30 sec, but not more than 15 times + since too much traffic may take too long to test. + ''' + interval_time = random.uniform(2, 3) + count = count+1 + time.sleep(interval_time) + client_socket.shutdown(socket.SHUT_WR) + received_data = b'' + while True: + data = client_socket.recv(1024) + if not data: + break + received_data += data + logging.info("Server response from port {}.".format(port)) + if received_data == payload.encode()*count: + results.append("{}:PASS".format(port)) + else: + results.append("{}:FAIL".format(port)) + except socket.error as e: + logging.error("{} on port {}".format(e, port)) + results.append("{}:ERROR".format(port)) + except Exception as e: + logging.error("{}: An unexpected error occurred for port {}" + .format(e, port)) + results.append("{}:ERROR".format(port)) + + +if __name__ == "__main__": + """ + TCP Ping Test + + This script performs a TCP ping test between a server and multiple + client ports. + The server listens on a range of ports, and the clients connect to + these ports to send a payload and receive a response from the server. + + Usage: + - To run as a server: ./script.py server -p -e + - To run as a client: ./script.py client -H -p + -e -P + + Arguments: + - mode (str): Specify whether to run as a server or client. + - host (str): Server host IP (client mode). This is mandatory arg. + - port (int): Starting port for the server or server port for the client. + Default is 1024. + - payload (int): Payload size in KB for the client. Default is 64. + - end_port (int): Ending port for the server. Default is 1223. + + Server Mode: + - The server listens on a range of ports concurrently, handling + incoming connections and send the received data back to client. + + Client Mode: + - The client connects to a range of server ports, + sending a payload and validating the received response. + The script logs pass, fail, or error status for each port. + """ + + parser = argparse.ArgumentParser( + description="Client-server with payload check on multiple ports") + + subparsers = parser.add_subparsers(dest="mode", + help="Run as server or client") + + # Subparser for the server command + server_parser = subparsers.add_parser("server", help="Run as server") + server_parser.add_argument("-p", "--port", + type=int, default=1024, + help="Starting port for the server") + server_parser.add_argument("-e", "--end-port", type=int, default=1223, + help="Ending port for the server") + + # Subparser for the client command + client_parser = subparsers.add_parser("client", help="Run as client") + client_parser.add_argument("-H", "--host", required=True, + help="Server host (client mode)") + client_parser.add_argument("-p", "--port", type=int, default=1024, + help="Starting port for the client") + client_parser.add_argument("-P", "--payload", type=int, default=64, + help="Payload size in KB (client mode)") + client_parser.add_argument("-e", "--end-port", type=int, default=1223, + help="Ending port for the client") + args = parser.parse_args() + + done_event = threading.Event() + results = [] + + # Ramp up time to wait until all ports are connected before + # starting to send the payload. + start_time = datetime.now() + timedelta(seconds=30) + + if args.mode == "server": + server(args.port, args.end_port) + elif args.mode == "client": + payload = 'A' * (args.payload * 1024) + client(args.host, args.port, args.end_port, + payload, done_event, start_time) + + # Wait for all threads to finish + done_event.wait() + + fail_port = list(filter(lambda x: "FAIL" in x, results)) + error_port = list(filter(lambda x: "ERROR" in x, results)) + if not (fail_port or error_port): + logging.info("TCP connections test pass!") + else: + if fail_port: + for x in fail_port: + logging.error("Fail on port {}.".format(x.split(":")[0])) + raise RuntimeError("TCP payload test fail!") + if error_port: + for x in error_port: + logging.error("Not able to connect on port {}." + .format(x.split(":")[0])) + raise RuntimeError("TCP connection fail!") diff --git a/checkbox-provider-ce-oem/bin/tcpecho_stress.sh b/checkbox-provider-ce-oem/bin/tcpecho_stress.sh new file mode 100755 index 0000000..6a495a8 --- /dev/null +++ b/checkbox-provider-ce-oem/bin/tcpecho_stress.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +get_active_interfaces() { + local net_state=$1 + # Get current network state + ip -o link show | grep 'state UP' | awk -F ': ' '{print $2}' > "$net_state" +} + +disable_net() { + local target_interface=$1 + local default_net_state=$2 + # Disable all network interfaces that are not under test + while IFS= read -r line; do + if [[ "$line" != *"$target_interface"* ]]; then + echo "Attempting to disable $line" + ip link set down dev "$line" + sleep 3 + fi + done < "$default_net_state" +} + +restore_net() { + local target_interface=$1 + local default_net_state=$2 + # Restore all network interfaces state + while IFS= read -r line; do + if [[ "$line" != *"$target_interface"* ]]; then + echo "Attempting to restore $line" + ip link set up dev "$line" + sleep 3 + fi + done < "$default_net_state" +} + +check_resote_net() { + for ((i=1; i <= 5; i++)) + do + net_check=0 + restore_net "$src_if" "$original_net_state" + sleep 20 # wait for every network interface up and get IP before exit + get_active_interfaces "$current_net_state" + if diff "$original_net_state" "$current_net_state" > /dev/null; then + echo "Info: All network states are restored successfully." + break + else + echo "Error: Not all network states are restored." + echo "Info: Trying again." + net_check=1 + fi + done + if [ "$net_check" -ne 0 ]; then + echo "Error: Not able to restore network states." + exit 1 + fi +} + + +tcp_echo() { + local target=$1 + local port=$2 + local loop=$3 + local file=$4 + local inloop="" + inloop=1000 + if [ "$loop" -lt "$inloop" ]; then + group_loop=1 + inloop="$loop" + # Deal with the loop is not multiple of 1000. + elif [ $((loop % inloop)) == 0 ]; then + group_loop=$((loop / inloop)) + else + group_loop=$((loop / inloop + 1)) + fi + status=0 + # Dividing the whole test into a small test group, to prevent the cost to much effort to get a failure result. + for ((x=1; i <= "$group_loop"; x++)) + do + for ((i=1; i <= "$inloop"; i++)) + do + # Redirect stdout and stderr to a variable while suppressing errors + # Will use BASH /dev/tcp device to handle TCP socket. + # Ref: https://tiswww.case.edu/php/chet/bash/bashref.html + echo=$( (timeout 1 echo "ID-$x-$i: TCP echo test!" > /dev/tcp/"$target"/"$port") 2>&1 ) + # It will get empty value if the TCP echo success. + if [[ -n "$echo" ]]; then + echo "ID-$x-$i: TCP echo to $target port $port Failed!" | tee -a "$file" + echo "$echo" >> "$file" + status=1 + fi + done + # Stop testing if group test failed + if [ "$status" -ne 1 ]; then + break + fi + done + return "$status" +} + +main() { + echo "Info: Attempting to test TCP echo stress ..." + echo "Info: Disabling network interfaces that not under test ..." + original_net_state=$(mktemp) + current_net_state=$(mktemp) + get_active_interfaces "$original_net_state" + disable_net "$src_if" "$original_net_state" + echo "Info: Checking if target is avaliable ..." + start_time=$(date +%s) + for ((i=1; i <= 5; i++)) + do + ping_state=0 + if ping -I "$src_if" "$dst_ip" -c 3; then + echo "Info: target is avaliable!" + break + else + echo "Error: Retry ping!" + ping_state=1 + sleep 5 + fi + done + if [ "$ping_state" -ne 0 ]; then + echo "Error: target $dst_ip is unavaliable" + echo "Info: Restore default network ..." + check_resote_net + exit 1 + fi + echo "Info: Starting to test TCP ping stress." + echo "Info: It will take time so please be patient." + echo "Info: Will print out log when failed." + if tcp_echo "$dst_ip" "$dst_port" "$loop" "$file"; then + echo "Info: TCP stress test completed for $loop times!" + else + echo "Error: TCP stress test failed." + echo "Info: Please refer to $file for more detail!" + fi + end_time=$(date +%s) + interval=$(("$end_time" - "$start_time")) + hours=$((interval / 3600)) + minutes=$((interval % 3600 / 60)) + seconds=$((interval % 60)) + echo "Time interval: $hours hours, $minutes minutes, $seconds seconds" + echo "Info: Restore default network ..." + check_resote_net + if [ "$status" -ne 0 ]; then + exit 1 + fi +} + +help_function() { + echo "This script is uses for TCP echo stress test." + echo "Run nc command on the server before you start to test" + echo "The following command can listen on certain port and direct message to log file." + echo " $ nc -lk -p {port} | tee test.log" + echo "Usage: tcpping.sh -s {src_eth_interface} -i {dst_IP} -p {dst_port} -l {num_of_loop} -o {file_to_log}" + echo -e "\t-s Source ethernet interface. e.g. eth0 or pfe0" + echo -e "\t-i Destination server IP address." + echo -e "\t-p Destination server port number, should be one of number in range 1024 to 65535" + echo -e "\t-l Number of the test loop." + echo -e "\t-o Output the fail log to file" +} + +while getopts "s:i:p:l:o:" opt; do + case "$opt" in + s) src_if="$OPTARG" ;; + i) dst_ip="$OPTARG" ;; + p) dst_port="$OPTARG" ;; + l) loop="$OPTARG" ;; + o) file="$OPTARG" ;; + ?) help_function ;; + esac +done + +if [[ -z "$src_if" || -z "$dst_ip" || -z "$dst_port" || -z "$loop" || -z "$file" ]]; then + echo "Error: Source network interface, Destination IP address,port,\ + Number of test loop and the output file are needed!" + help_function + exit 1 +fi + +main \ No newline at end of file diff --git a/checkbox-provider-ce-oem/units/ethernet/jobs.pxu b/checkbox-provider-ce-oem/units/ethernet/jobs.pxu new file mode 100644 index 0000000..070684a --- /dev/null +++ b/checkbox-provider-ce-oem/units/ethernet/jobs.pxu @@ -0,0 +1,42 @@ +unit: template +template-resource: com.canonical.certification::device +template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN' +template-engine: jinja2 +template-unit: job +imports: + from com.canonical.certification import device + from com.canonical.plainbox import manifest +requires: manifest.has_tcp_echo_stress_server == 'True' +id: ce-oem-ethernet/tcp-echo-stress-{{ interface }} +plugin: shell +user: root +category_id: com.canonical.plainbox::stress +_summary: Check if TCP echo via {{ interface }} without error. +_description: + This job will use BASH to handle TCP socket via /dev/tcp. + Need a server to run the following command before running the test. + $ nc -lk -p {port_num} +environ: TCP_ECHO_SERVER_IP TCP_ECHO_SERVER_PORT TCP_ECHO_LOOP_ITERATIONS +estimated_duration: 10h +flags: also-after-suspend +command: tcpecho_stress.sh -s {{ interface }} -i "$TCP_ECHO_SERVER_IP" -p "$TCP_ECHO_SERVER_PORT" -l "$TCP_ECHO_LOOP_ITERATIONS" -o "${PLAINBOX_SESSION_SHARE}"/tcp_echo.log + +id: ce-oem-ethernet/tcp-multi-connections +plugin: shell +user: root +category_id: com.canonical.plainbox::ethernet +_summary: Check if the system can handle multiple connections on TCP without error. +_description: + This job will connect to server listened ports(200 ports in total), + and send the payload(64KB) for 10 times of each port. This job will + send the payload after all ports connection is established. + Need a server to run the following command before running the test. + e.g. Run a server to listen on port range from 1024 to 1223. + $ tcp_multi_connections.py server -p 1024 -e 1223 +environ: TCP_MULTI_CONNECTIONS_SERVER_IP TCP_MULTI_CONNECTIONS_START_PORT TCP_MULTI_CONNECTIONS_END_PORT TCP_MULTI_CONNECTIONS_PAYLOAD_SIZE +estimated_duration: 300 +flags: also-after-suspend +requires: manifest.has_tcp_multi_connection_server == 'True' +imports: from com.canonical.plainbox import manifest +command: + tcp_multi_connections.py client -H "$TCP_MULTI_CONNECTIONS_SERVER_IP" -p "$TCP_MULTI_CONNECTIONS_START_PORT" -e "$TCP_MULTI_CONNECTIONS_END_PORT" -P "$TCP_MULTI_CONNECTIONS_PAYLOAD_SIZE" diff --git a/checkbox-provider-ce-oem/units/ethernet/manifest.pxu b/checkbox-provider-ce-oem/units/ethernet/manifest.pxu new file mode 100644 index 0000000..cc61c89 --- /dev/null +++ b/checkbox-provider-ce-oem/units/ethernet/manifest.pxu @@ -0,0 +1,9 @@ +unit: manifest entry +id: has_tcp_multi_connection_server +_name: Has the TCP multi-connection server been set up? +value-type: bool + +unit: manifest entry +id: has_tcp_echo_stress_server +_name: Has the TCP echo stress server been set up? +value-type: bool diff --git a/checkbox-provider-ce-oem/units/ethernet/test-plan.pxu b/checkbox-provider-ce-oem/units/ethernet/test-plan.pxu new file mode 100644 index 0000000..a191f2d --- /dev/null +++ b/checkbox-provider-ce-oem/units/ethernet/test-plan.pxu @@ -0,0 +1,34 @@ +id: ce-oem-ethernet-tcp-stress +unit: test plan +_name: TCP stress test plan +_description: TCP stress +estimated_duration: 60h +bootstrap_include: + com.canonical.certification::device +include: + ce-oem-ethernet/tcp-echo-stress-.* + +id: ce-oem-tcp-full +unit: test plan +_name: TCP connection tests +_description: TCP connection test of the device +include: +nested_part: + ce-oem-ethernet-tcp-automated + after-suspend-ce-oem-ethernet-tcp-automated + +id: ce-oem-ethernet-tcp-automated +unit: test plan +_name: TCP connection test plan +_description: TCP connection test +estimated_duration: 300 +include: + ce-oem-ethernet/tcp-multi-connections + +id: after-suspend-ce-oem-ethernet-tcp-automated +unit: test plan +_name: TCP connection test plan +_description: TCP connection test +estimated_duration: 300 +include: + after-suspend-ce-oem-ethernet/tcp-multi-connections diff --git a/checkbox-provider-ce-oem/units/test-plan-ce-oem.pxu b/checkbox-provider-ce-oem/units/test-plan-ce-oem.pxu index 6486bb8..62cefb9 100644 --- a/checkbox-provider-ce-oem/units/test-plan-ce-oem.pxu +++ b/checkbox-provider-ce-oem/units/test-plan-ce-oem.pxu @@ -73,6 +73,7 @@ nested_part: ce-oem-crypto-automated ce-oem-optee-automated ce-oem-socketcan-stress-automated + ce-oem-ethernet-tcp-automated com.canonical.certification::eeprom-automated com.canonical.certification::rtc-automated @@ -130,6 +131,7 @@ nested_part: after-suspend-ce-oem-crypto-automated after-suspend-ce-oem-optee-automated after-suspend-ce-oem-socketcan-stress-automated + after-suspend-ce-oem-ethernet-tcp-automated com.canonical.certification::after-suspend-eeprom-automated com.canonical.certification::after-suspend-rtc-automated @@ -142,3 +144,4 @@ estimated_duration: 3600 include: nested_part: ce-oem-cold-boot-stress-test-by-pdu + ce-oem-ethernet-tcp-stress