From 142b64b3e16621b27bf0c0c952ff7c3b751cf613 Mon Sep 17 00:00:00 2001 From: AlexandreSinger Date: Thu, 30 May 2024 23:48:41 -0400 Subject: [PATCH] Squashed 'libs/EXTERNAL/libtatum/' changes from d68a6b3ec..28e4e0ae0 28e4e0ae0 Use more robust checksum based checks to verify test file downloads 12f41e056 Add titan23 benchmarks to regression tests eda613885 Remove deprecated 'sudo' option from TravisCI config 85929e0f9 Use .xz compressed version of mcnc20 benchmarks 269840e91 Remove pathlib dependency in reg_test.py e6e2efc85 Include VTR and titan_other benchmarks in CI 1355d6ee8 Include titan_other benchmarks in regression tests 565b6b7c7 Tweak reg_test.py to skip downloading benchmarks if already existing 40fc409bb Add support for VTR benchmark regression tests git-subtree-dir: libs/EXTERNAL/libtatum git-subtree-split: 28e4e0ae0c1b9ff0120394838b5fdf4ef08e5c9a --- .gitignore | 8 +++ .travis.yml | 7 ++- scripts/reg_test.py | 117 +++++++++++++++++++++++++++++++++----------- 3 files changed, 99 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index b9b0d1e277d..9e2c0ca5323 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,11 @@ callgrind* *.rpt *.dot *.echo + +#Test files +test/mcnc20 +test/vtr +test/titan_other +test/titan23 +*.tar +*.tar.gz diff --git a/.travis.yml b/.travis.yml index 347c699349f..d86da98ba5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,16 @@ language: cpp dist: trusty #Ubuntu 14.04 by default -sudo: false #Use container based infrastructure matrix: include: #Extensive testing for base compiler - - env: TESTS="basic mcnc20" TATUM_EXECUTION_ENGINE=auto MATRIX_EVAL="CC=gcc-5 CXX=g++-5" + - env: TESTS="basic mcnc20 vtr titan_other" TATUM_EXECUTION_ENGINE=auto MATRIX_EVAL="CC=gcc-5 CXX=g++-5" addons: { apt: { packages: ["cmake", "g++-5", "libtbb-dev"], sources: ["ubuntu-toolchain-r-test"] } } - - env: TESTS="basic mcnc20" TATUM_EXECUTION_ENGINE=serial MATRIX_EVAL="CC=gcc-5 CXX=g++-5" + - env: TESTS="basic mcnc20 vtr titan_other" TATUM_EXECUTION_ENGINE=serial MATRIX_EVAL="CC=gcc-5 CXX=g++-5" addons: { apt: { packages: ["cmake", "g++-5", "libtbb-dev"], sources: ["ubuntu-toolchain-r-test"] } } - - env: TESTS="basic mcnc20" TATUM_EXECUTION_ENGINE=tbb MATRIX_EVAL="CC=gcc-5 CXX=g++-5" + - env: TESTS="basic mcnc20 vtr titan_other" TATUM_EXECUTION_ENGINE=tbb MATRIX_EVAL="CC=gcc-5 CXX=g++-5" addons: { apt: { packages: ["cmake", "g++-5", "libtbb-dev"], sources: ["ubuntu-toolchain-r-test"] } } #Simple testing for other compilers diff --git a/scripts/reg_test.py b/scripts/reg_test.py index c034942473a..b9db05cd27a 100755 --- a/scripts/reg_test.py +++ b/scripts/reg_test.py @@ -10,22 +10,16 @@ import math import subprocess import shutil - +import hashlib TATUM_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) tests = { 'basic': [os.path.join(TATUM_ROOT, 'test', 'basic')], - 'mcnc20': ["http://www.eecg.utoronto.ca/~kmurray/tatum/golden_results/mcnc20_tatum_golden.tar.gz"], - #'vtr': [ - - #], - #'titan_other': [ - - #], - #'titan': [ - - #], + 'mcnc20': ["http://www.eecg.utoronto.ca/~kmurray/tatum/golden_results/mcnc20_tatum_golden.tar"], + 'vtr': ["http://www.eecg.utoronto.ca/~kmurray/tatum/golden_results/vtr_tatum_golden.tar"], + 'titan_other': ["http://www.eecg.utoronto.ca/~kmurray/tatum/golden_results/titan_other_tatum_golden.tar"], + 'titan23': ["http://www.eecg.utoronto.ca/~kmurray/tatum/golden_results/titan23_tatum_golden.tar"], #'tau15_unit_delay': [ #], @@ -68,6 +62,11 @@ def parse_args(): default=False, action='store_true') + parser.add_argument("-f", "--force", + help="Force download even if test files already found", + default=False, + action='store_true') + return parser.parse_args() def main(): @@ -83,7 +82,7 @@ def main(): for test_url in tests[test_name]: work_dir = tempfile.mkdtemp(prefix="tatum_reg_test") try: - test_files = download_extract_test(args, work_dir, test_url) + test_files = download_extract_test(args, test_name, test_url) num_failures += run_test(args, work_dir, test_name, test_files) finally: if not args.debug: @@ -123,16 +122,28 @@ def run_single_test(args, work_dir, test_file): if args.tatum_nworkers: cmd += ['--num_workers', str(args.tatum_nworkers)] - cmd += [test_file] print(" {}".format(test_file), end='') - if args.debug: - print() - print(" cmd: {} log: {}".format(' '.join(cmd), log)) + if test_file.endswith(".xz"): + + cmd += ["-"] #Read from stdin + + cmd_str = "xzcat {} | {} > {}".format(test_file, " ".join(cmd), log) + + if args.debug: + print(cmd_str) + + retcode = os.system(cmd_str) + else: + cmd += [test_file] #Read file directly - with open(log, 'w') as outfile: - retcode = subprocess.call(cmd, stdout=outfile, stderr=outfile) + if args.debug: + print() + print(" cmd: {} log: {}".format(' '.join(cmd), log)) + + with open(log, 'w') as outfile: + retcode = subprocess.call(cmd, stdout=outfile, stderr=outfile) if retcode == 0: print(" PASSED") @@ -147,35 +158,53 @@ def run_single_test(args, work_dir, test_file): return num_failed -def download_extract_test(args, work_dir, test_url): +def download_extract_test(args, test_name, test_url): test_files = [] - if '.tar' in test_url: + if 'tar' in test_url: #A tar file of benchmark files - benchmark_tar = os.path.join(work_dir, os.path.basename(test_url)) + benchmark_tar = os.path.join(os.path.join(TATUM_ROOT, os.path.basename(test_url))) - get_url(test_url, benchmark_tar) + new_tar = get_url(args, test_url, benchmark_tar) + + test_files_dir = os.path.join(TATUM_ROOT, "test") + + if new_tar or args.force: + + print("Extracting test files to {}".format(test_files_dir)) + with tarfile.TarFile.open(benchmark_tar, mode="r|*") as tar_file: + tar_file.extractall(path=test_files_dir) + else: + print("Skipping file extraction".format(test_files_dir)) - with tarfile.TarFile.open(benchmark_tar, mode="r|*") as tar_file: - tar_file.extractall(path=work_dir) - test_files += glob.glob("{}/*.tatum".format(work_dir)) - test_files += glob.glob("{}/*/*.tatum".format(work_dir)) + test_files += glob.glob("{}/{}/*.tatum*".format(test_files_dir, test_name)) else: #A directory of benchmark files test_files += glob.glob("{}/*.tatum".format(test_url)) - test_files += glob.glob("{}/*/*.tatum".format(test_url)) return test_files -def get_url(url, filename): +def get_url(args, url, filename): + if not args.force and os.path.exists(filename): + print("Found existing file {}, checking if hash matches".format(filename)) + file_matches = check_hash_match(args, url, filename) + + if file_matches: + print("Existing file {} matches, skipping download".format(filename)) + return False + else: + print("Existing file {} contents differ, re-downloading".format(filename)) + if '://' in url: download_url(url, filename) else: shutl.copytree(url, filename) + return True + def download_url(url, filename): """ Downloads the specifed url to filename @@ -191,9 +220,39 @@ def download_progress_callback(block_num, block_size, expected_size): progress_increment = int(math.ceil(total_blocks / 100)) if block_num % progress_increment == 0: - print(".", end='', flush=False) + print(".", end='', flush=True) if block_num*block_size >= expected_size: print("") +def check_hash_match(args, url, filename): + checksum_url = url + ".sha256" + try: + web_hash = urllib.request.urlopen(checksum_url).read() + except urllib.error.HTTPError as e: + print("Failed to find expected SHA256 checksum at {} (reason '{}')".format(checksum_url, e)) + return False + + local_hash = hash_file(filename) + + web_digest_bytes = web_hash.split()[0] + local_digest_bytes = str.encode(local_hash) + + if web_digest_bytes == local_digest_bytes: + return True + + return False + +def hash_file(filepath): + BUF_SIZE = 65536 + sha256 = hashlib.sha256() + with open(filepath, "rb") as f: + while True: + data = f.read(BUF_SIZE) + if not data: + break + sha256.update(data) + + return sha256.hexdigest() + if __name__ == "__main__": main()