From c6f246ad23884840c96f1c225344c1d23a53b434 Mon Sep 17 00:00:00 2001 From: Sam Xi Date: Sat, 25 Jul 2020 21:43:47 -0700 Subject: [PATCH] tests: Migrate from Travis to CircleCI. CircleCI will give us better scalability for larger integration tests than Travis at lower cost. The CircleCI configuration includes build, unit testing, and downloading a recently built gem5-aladdin binary so that we can run gem5 simulations in the future as part of the CI flow. We don't produce JUnit reports as that causes the tests to OOM. Issue #25. TESTED=verified CircleCI pipeline passes. --- .circleci/config.yml | 27 ++++++++++ .circleci/download_artifacts.py | 88 +++++++++++++++++++++++++++++++++ .travis.yml | 56 --------------------- README.md | 2 +- make/Makefile.native | 2 + 5 files changed, 118 insertions(+), 57 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .circleci/download_artifacts.py delete mode 100644 .travis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..cb813a89 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,27 @@ +version: 2.1 +jobs: + build: + docker: + - image: xyzsam/smaug:latest + + environment: + SMAUG_HOME: /root/project + steps: + - checkout + - run: + name: Checkout dependencies + command: git submodule update --init --recursive + - run: + name: Build + command: | + make all -j2 + make test -j2 + - run: + name: Run unit tests + command: | + export PYTHONPATH=$SMAUG_HOME:$PYTHONPATH + make test-run + - run: + name: Download latest gem5-aladdin binary + command: + python .circleci/download_artifacts.py --api_token=${GEM5_ALADDIN_BUILD_ARTIFACT_TOKEN} --project=gem5-aladdin --artifact_name=gem5.opt --user=${USER} --download_loc=/tmp --filter=${BUILD_ARTIFACT_FILTER} --branch=${BUILD_ARTIFACT_BRANCH} diff --git a/.circleci/download_artifacts.py b/.circleci/download_artifacts.py new file mode 100644 index 00000000..36e85002 --- /dev/null +++ b/.circleci/download_artifacts.py @@ -0,0 +1,88 @@ +"""Downloads build artifacts via the CircleCI API.""" + +import argparse +import os +import json +import six +import subprocess +import sys + +def query_artifacts(api_token, user, project, branch, filter): + """Queries CircleCI for a list of build artifacts. + + All parameters are strings. The returned value is a JSON object with the + following attributes: + "path": Path to the artifact in the project, relative to the working + directory. + "pretty_path": Same as path. + "node_index": Ignored. + "url": The URL at which the artifact is located. + """ + url = "https://circleci.com/api/v1.1/project/github/%(user)s/%(project)s/latest/artifacts?branch=%(branch)s&filter=%(filter)s" % { + "user": user, + "project": project, + "branch": branch, + "filter": filter, + } + args = ["curl", "-H'Circle-Token: %s'" % api_token, url] + print(" ".join(args)) + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + print("Stdout: %s", stdout) + print("Stderr: %s", stderr) + if proc.returncode != 0: + raise OSError("artifact lookup failed") + artifacts = json.loads(stdout) + return artifacts + + +def download_artifacts(artifacts, artifact_name, download_loc): + """Downloads matching artifacts to the desired location. + + artifacts: List of artifacts from the CircleCI API. + artifact_name: The basename of the artifacts to download. + download_loc: The directory where the objects will be downloaded to. + """ + for artifact in artifacts: + if os.path.basename(artifact["path"]) == artifact_name: + proc = subprocess.Popen(["wget", "-P", download_loc, artifact["url"]]) + stdout, stderr = proc.communicate() + if stdout is not None: + print(six.ensure_str(stdout)) + if stderr is not None: + print(six.ensure_str(stderr)) + if proc.returncode != 0: + raise OSError("failed to download artifact") + + +def main(): + parser = argparse.ArgumentParser( + "Download CircleCI artifacts for SMAUG dependencies") + parser.add_argument("--artifact_name", type=str, required=True, + help="Base filename of the object to download.") + parser.add_argument("--download_loc", type=str, required=True, + help="Directory to store the downloaded artifact.") + parser.add_argument("--api_token", type=str, required=True, + help="Secret API token for access to CircleCI APIs.") + parser.add_argument("--project", type=str, required=True, + help="Name of the project to download from") + parser.add_argument("--user", type=str, default="harvard-acc", + help="CircleCI username.") + parser.add_argument("--branch", type=str, default="master", + help="Branch from which the artifact was built.") + parser.add_argument("--filter", type=str, default="successful", + help="Filter on which builds to return.") + args = parser.parse_args() + artifacts = query_artifacts(args.api_token, + args.user, + args.project, + args.branch, + args.filter) + if "message" in artifacts: + print("Artifact query failed:", artifacts["message"]) + sys.exit(1) + download_artifacts(artifacts, args.artifact_name, args.download_loc) + + +if __name__ == "__main__": + main() diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5c4bc390..00000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ -dist: bionic - -before_install: - - sudo apt-get -qq update - - sudo apt-get -qq install -y libprotobuf-dev protobuf-compiler - - sudo apt-get -qq install -y libboost-graph-dev libboost-program-options-dev - - sudo apt-get -qq install -y python3 python3-pip python3-setuptools - # Install a newer version of Protobuf. - - | - pushd /tmp - wget -q https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protobuf-all-3.11.4.tar.gz - tar -xzf protobuf-all-3.11.4.tar.gz - cd protobuf-3.11.4 - ./configure > /dev/null - make -j2 > /dev/null - sudo make install - sudo ldconfig - popd - # Install Python 3.6 before installing required packages. - - pip3 install --upgrade pip - - sudo update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 - - sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 - - pip install --progress-bar off -r requirements.txt - # Clone LLVM-Tracer and gem5-Aladdin without building. SMAUG build requires - # copying a few files from these two repos. - - git clone https://github.com/ysshao/LLVM-Tracer - - | - git clone https://github.com/harvard-acc/gem5-aladdin - cd gem5-aladdin - git submodule init - git submodule update - -env: - - | - SMAUG_HOME=$TRAVIS_BUILD_DIR - ALADDIN_HOME=$TRAVIS_BUILD_DIR/gem5-aladdin/src/aladdin - TRACER_HOME=$TRAVIS_BUILD_DIR/LLVM-Tracer - BOOST_ROOT=/usr/local - PROTOC=/usr/local/bin/protoc - PYTHONPATH=$SMAUG_HOME:$PYTHONPATH - -language: c++ -compiler: g++ - -script: - - | - cd $TRAVIS_BUILD_DIR - ./run_travis.sh - -notifications: - email: - on_success: never - on_failure: always - recipients: - - slxi1202@gmail.com - - yaoyuannnn@gmail.com diff --git a/README.md b/README.md index 8b446104..4a5f0d1a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ SMAUG: Simulating Machine Learning Accelerators Using gem5-Aladdin ================================================================== -[![build status](https://travis-ci.org/harvard-acc/smaug.svg?branch=master)](https://travis-ci.org/harvard-acc/smaug) +[![harvard-acc](https://circleci.com/gh/harvard-acc/smaug.svg?style=shield)](https://circleci.com/gh/harvard-acc/smaug) SMAUG is a library for building and simulating neural networks, written to work with gem5-Aladdin. It supports the basic layer types and basic activation diff --git a/make/Makefile.native b/make/Makefile.native index 38719ce3..5c2cef8c 100644 --- a/make/Makefile.native +++ b/make/Makefile.native @@ -4,6 +4,8 @@ include make/Makefile.common .PHONY: all tests clean run-tests +SHELL:=/bin/bash + #################################### #### COMPILATION FLAGS #### ####################################