From f901d0c56a9db4dadc95725273b0a1ef7f8a8866 Mon Sep 17 00:00:00 2001 From: Andrew Quijano Date: Tue, 10 Dec 2024 14:42:02 -0500 Subject: [PATCH] Centralize all dependancy information and also update some packages --- .github/workflows/parallel_tests.yml | 8 +-- Dockerfile | 70 +++++++++++-------- panda/debian/.gitignore | 3 +- panda/dependencies/ubuntu_18.04_base.txt | 3 +- panda/dependencies/ubuntu_20.04_base.txt | 3 +- panda/dependencies/ubuntu_22.04_base.txt | 3 +- .../syscalls2/scripts/requirements2.txt | 2 +- panda/python/core/.gitignore | 1 + panda/scripts/install_ubuntu.sh | 19 ++--- 9 files changed, 66 insertions(+), 46 deletions(-) diff --git a/.github/workflows/parallel_tests.yml b/.github/workflows/parallel_tests.yml index d58d93e7948..4048d41d0ba 100644 --- a/.github/workflows/parallel_tests.yml +++ b/.github/workflows/parallel_tests.yml @@ -20,9 +20,9 @@ on: jobs: test_installer: # test install_ubuntu.sh - runs-on: panda-arc # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet) + runs-on: panda-arc container: - image: ubuntu:20.04 + image: ubuntu:22.04 steps: - name: Update run: apt-get -qq update -y @@ -31,9 +31,9 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: '3.10' - name: Install Python dev headers - run: apt-get -qq install -y libpython3.9-dev + run: apt-get -qq install -y libpython3.10-dev - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - name: Lint PyPANDA with flake8 run: | diff --git a/Dockerfile b/Dockerfile index da5748372f1..718e3e53795 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ ARG BASE_IMAGE="ubuntu:20.04" ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,aarch64-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu,mips64-softmmu,mips64el-softmmu" -ARG LIBOSI_VERSION="v0.1.7" +ARG INSTALL_PREFIX="/usr/local" ### BASE IMAGE FROM $BASE_IMAGE as base ARG BASE_IMAGE +ARG INSTALL_PREFIX # Copy dependencies lists into container. We copy them all and then do a mv because # we need to transform base_image into a windows compatible filename which we can't @@ -16,22 +17,19 @@ RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt & # Base image just needs runtime dependencies RUN [ -e /tmp/base_dep.txt ] && \ apt-get -qq update && \ - DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \ + DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl jq $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \ apt-get clean ### BUILD IMAGE - STAGE 2 FROM base AS builder ARG BASE_IMAGE ARG TARGET_LIST -ARG LIBOSI_VERSION +ARG INSTALL_PREFIX RUN [ -e /tmp/build_dep.txt ] && \ apt-get -qq update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/build_dep.txt | grep -o '^[^#]*') && \ apt-get clean && \ - python3 -m pip install --upgrade --no-cache-dir pip && \ - python3 -m pip install --upgrade --no-cache-dir "cffi>1.14.3" && \ - python3 -m pip install --upgrade --no-cache-dir "capstone" && \ curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal # Then install capstone from source @@ -43,23 +41,30 @@ RUN cd /tmp && \ ENV PATH="/root/.cargo/bin:${PATH}" # install libosi -RUN cd /tmp && curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_$(echo "$BASE_IMAGE" | awk -F':' '{print $2}').deb && dpkg -i /tmp/libosi_$(echo "$BASE_IMAGE" | awk -F':' '{print $2}').deb +RUN cd /tmp && \ + BASE_IMAGE_VERSION=$(echo "$BASE_IMAGE" | awk -F':' '{print $2}') && \ + LIBOSI_VERSION=$(curl -s https://api.github.com/repos/panda-re/libosi/releases/latest | jq -r .tag_name) && \ + curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_${BASE_IMAGE_VERSION}.deb && \ + dpkg -i /tmp/libosi_${BASE_IMAGE_VERSION}.deb && \ + rm -rf /tmp/libosi_${BASE_IMAGE_VERSION}.deb # Build and install panda # Copy repo root directory to /panda, note we explicitly copy in .git directory # Note .dockerignore file keeps us from copying things we don't need +# PyPANDA needs CFFI from pip (the version in apt is too old) COPY . /panda/ COPY .git /panda/ +RUN pip install -r /panda/panda/python/core/requirements.txt # Note we diable NUMA for docker builds because it causes make check to fail in docker RUN git -C /panda submodule update --init dtc && \ - git -C /panda rev-parse HEAD > /usr/local/panda_commit_hash && \ + git -C /panda rev-parse HEAD > ${INSTALL_PREFIX}/panda_commit_hash && \ mkdir /panda/build && cd /panda/build && \ python3 -m pip install setuptools_scm && \ python3 -m setuptools_scm -r .. --strip-dev 2>/dev/null >/tmp/savedversion && \ /panda/configure \ --target-list="${TARGET_LIST}" \ - --prefix=/usr/local \ + --prefix=${INSTALL_PREFIX} \ --disable-numa \ --enable-llvm && \ rm -rf /panda/.git @@ -69,9 +74,9 @@ RUN PRETEND_VERSION=$(cat /tmp/savedversion) make -C /panda/build -j "$(nproc)" #### Develop setup: panda built + pypanda installed (in develop mode) - Stage 3 FROM builder as developer +ARG INSTALL_PREFIX RUN cd /panda/panda/python/core && \ PRETEND_VERSION=$(cat /tmp/savedversion) python3 setup.py develop && \ - ldconfig && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \ cd /panda && \ ( git config --get-regexp http > /dev/null && \ @@ -81,17 +86,17 @@ WORKDIR /panda/ #### Install PANDA + pypanda from builder - Stage 4 FROM builder as installer +ARG INSTALL_PREFIX RUN make -C /panda/build install && \ - rm -r /usr/local/lib/panda/*/cosi \ - /usr/local/lib/panda/*/cosi_strace \ - /usr/local/lib/panda/*/gdb \ - /usr/local/lib/panda/*/snake_hook \ - /usr/local/lib/panda/*/rust_skeleton + rm -r ${INSTALL_PREFIX}/lib/panda/*/cosi \ + ${INSTALL_PREFIX}/lib/panda/*/cosi_strace \ + ${INSTALL_PREFIX}/lib/panda/*/gdb \ + ${INSTALL_PREFIX}/lib/panda/*/snake_hook \ + ${INSTALL_PREFIX}/lib/panda/*/rust_skeleton # Install pypanda RUN cd /panda/panda/python/core && \ PRETEND_VERSION=$(cat /tmp/savedversion) python3 setup.py install -RUN python3 -m pip install --ignore-install pycparser && python3 -m pip install --force-reinstall --no-binary :all: cffi # Build a whl too RUN cd /panda/panda/python/core && \ PRETEND_VERSION=$(cat /tmp/savedversion) python3 setup.py bdist_wheel @@ -103,40 +108,47 @@ RUN bash -c "ls $(pip show pandare | grep Location: | awk '{print $2}')/pandare/ # this layer is used to strip shared objects and change python data to be # symlinks to the installed panda data directory FROM installer as cleanup -RUN find /usr/local/lib/panda -name "*.so" -exec strip {} \; +ARG INSTALL_PREFIX +RUN find ${INSTALL_PREFIX}/lib/panda -name "*.so" -exec strip {} \; RUN PKG=`pip show pandare | grep Location: | awk '{print $2}'`/pandare/data; \ - rm -rf $PKG/pc-bios && ln -s /usr/local/share/panda $PKG/pc-bios; \ + rm -rf $PKG/pc-bios && ln -s ${INSTALL_PREFIX}/share/panda $PKG/pc-bios; \ for arch in `find $PKG -name "*-softmmu" -type d -exec basename {} \;` ; do \ ARCHP=$PKG/$arch; \ SARCH=`echo $arch | cut -d'-' -f 1`; \ rm $ARCHP/libpanda-$SARCH.so $ARCHP/llvm-helpers-$SARCH.bc; \ - ln -s /usr/local/share/panda/llvm-helpers-$SARCH.bc $ARCHP/llvm-helpers-$SARCH.bc1; \ - ln -s /usr/local/bin/libpanda-$SARCH.so $ARCHP/libpanda-$SARCH.so; \ + ln -s ${INSTALL_PREFIX}/share/panda/llvm-helpers-$SARCH.bc $ARCHP/llvm-helpers-$SARCH.bc1; \ + ln -s ${INSTALL_PREFIX}/bin/libpanda-$SARCH.so $ARCHP/libpanda-$SARCH.so; \ rm -rf $ARCHP/panda/plugins; \ - ln -s /usr/local/lib/panda/$SARCH/ $ARCHP/panda/plugins; \ + ln -s ${INSTALL_PREFIX}/lib/panda/$SARCH/ $ARCHP/panda/plugins; \ done ### Copy files for panda+pypanda from installer - Stage 5 FROM base as panda +ARG INSTALL_PREFIX +ARG TARGET_LIST # Include dependency lists for packager COPY --from=base /tmp/base_dep.txt /tmp COPY --from=base /tmp/build_dep.txt /tmp # Copy panda + libcapstone.so* + libosi libraries -COPY --from=cleanup /usr/local /usr/local +COPY --from=cleanup ${INSTALL_PREFIX} ${INSTALL_PREFIX} COPY --from=cleanup /usr/lib/libcapstone* /usr/lib/ -COPY --from=cleanup /lib/libosi.so /lib/libiohal.so /lib/liboffset.so /lib/ +COPY --from=cleanup /usr/lib/x86_64-linux-gnu/libosi.so /usr/lib/x86_64-linux-gnu/libiohal.so /lib/liboffset.so /usr/lib/x86_64-linux-gnu/ # Workaround issue #901 - ensure LD_LIBRARY_PATH contains the panda plugins directories -#ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu" -ENV LD_LIBRARY_PATH /usr/local/lib/python3.8/dist-packages/pandare/data/x86_64-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/i386-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/arm-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/ppc-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/mips-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/mipsel-softmmu/panda/plugins/ -#PANDA_PATH is used by rust plugins -ENV PANDA_PATH /usr/local/lib/python3.8/dist-packages/pandare/data +RUN LD_LIBRARY_PATH="" && \ + for arch in $(echo $TARGET_LIST | tr ',' ' '); do \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTALL_PREFIX}/lib/python3.8/dist-packages/pandare/data/${arch}/panda/plugins/"; \ + done && \ + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" > /tmp/ld_library_path +ENV LD_LIBRARY_PATH $(cat /tmp/ld_library_path) +# PANDA_PATH is used by rust plugins +ENV PANDA_PATH ${INSTALL_PREFIX}/lib/python3.8/dist-packages/pandare/data # Ensure runtime dependencies are installed for our libpanda objects and panda plugins RUN ldconfig && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \ - if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/libpanda-*.so | grep 'not found'); then exit 1; fi && \ - if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi + if (ldd ${INSTALL_PREFIX}/lib/python*/dist-packages/pandare/data/*-softmmu/libpanda-*.so | grep 'not found'); then exit 1; fi && \ + if (ldd ${INSTALL_PREFIX}/lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi \ No newline at end of file diff --git a/panda/debian/.gitignore b/panda/debian/.gitignore index 7bbf39a4c32..3277dbe3185 100644 --- a/panda/debian/.gitignore +++ b/panda/debian/.gitignore @@ -1 +1,2 @@ -panda.deb +*.deb +*.whl \ No newline at end of file diff --git a/panda/dependencies/ubuntu_18.04_base.txt b/panda/dependencies/ubuntu_18.04_base.txt index e4605e2ba9f..13328881c93 100644 --- a/panda/dependencies/ubuntu_18.04_base.txt +++ b/panda/dependencies/ubuntu_18.04_base.txt @@ -18,11 +18,12 @@ unzip # pyperipheral (only needed for armel) libpython3-dev -# pypanda dependencies +# panda python dependencies genisoimage libffi-dev python3-protobuf python3-colorama +python3-capstone # apt-rdepends qemu-system-common acl diff --git a/panda/dependencies/ubuntu_20.04_base.txt b/panda/dependencies/ubuntu_20.04_base.txt index 7b043e8c3ed..6d2eb6624d7 100644 --- a/panda/dependencies/ubuntu_20.04_base.txt +++ b/panda/dependencies/ubuntu_20.04_base.txt @@ -17,11 +17,12 @@ wget # pyperipheral (only needed for armel) libpython3-dev -# pypanda dependencies +# panda python dependencies genisoimage libffi-dev python3-protobuf python3-colorama +python3-capstone # Not sure what this one is needed for liblzo2-2 diff --git a/panda/dependencies/ubuntu_22.04_base.txt b/panda/dependencies/ubuntu_22.04_base.txt index ed4549b4cfe..ddf261d6979 100644 --- a/panda/dependencies/ubuntu_22.04_base.txt +++ b/panda/dependencies/ubuntu_22.04_base.txt @@ -17,11 +17,12 @@ wget # pyperipheral (only needed for armel) libpython3-dev -# pypanda dependencies +# panda python dependencies genisoimage libffi-dev python3-protobuf python3-colorama +python3-capstone # Not sure what this one is needed for liblzo2-2 diff --git a/panda/plugins/syscalls2/scripts/requirements2.txt b/panda/plugins/syscalls2/scripts/requirements2.txt index 0f591a2b388..f298d5d6b1b 100644 --- a/panda/plugins/syscalls2/scripts/requirements2.txt +++ b/panda/plugins/syscalls2/scripts/requirements2.txt @@ -1,2 +1,2 @@ -Jinja2==3.1.2 +Jinja2>=3.1.4 MarkupSafe==2.1.3 diff --git a/panda/python/core/.gitignore b/panda/python/core/.gitignore index 4eebd474f78..fa0ca2b31c2 100644 --- a/panda/python/core/.gitignore +++ b/panda/python/core/.gitignore @@ -8,3 +8,4 @@ data a __pycache__ *.egg-info +.eggs/ \ No newline at end of file diff --git a/panda/scripts/install_ubuntu.sh b/panda/scripts/install_ubuntu.sh index 73b853e7281..b2fb9c2205a 100755 --- a/panda/scripts/install_ubuntu.sh +++ b/panda/scripts/install_ubuntu.sh @@ -23,9 +23,14 @@ fi # Note package names should be consistent across Ubuntu versions. lsb_release --help &>/dev/null || $SUDO apt-get update -qq && $SUDO apt-get -qq install -y --no-install-recommends lsb-release git --help &>/dev/null || $SUDO apt-get -qq update && $SUDO apt-get -qq install -y --no-install-recommends git +$SUDO apt-get install -y --no-install-recommends curl jq # some globals -LIBOSI_VERSION="0.1.7" +# TODO: Why is curl -s failing in panda-arc? For now, I'll put a fallback version +LIBOSI_VERSION=$(curl -s https://api.github.com/repos/panda-re/libosi/releases/latest | jq -r .tag_name) +if [ -z "$LIBOSI_VERSION" ]; then + LIBOSI_VERSION="v0.1.10" +fi UBUNTU_VERSION=$(lsb_release -r | awk '{print $2}') PANDA_GIT="https://github.com/panda-re/panda.git" @@ -76,7 +81,6 @@ if [ $version -eq 18 ]; then $SUDO apt-get update fi - # Dependencies are for a major version, but the filenames include minor versions # So take our major version, find the first match in dependencies directory and run with it. # This will give us "./panda/dependencies/ubuntu:20.04" where ubuntu:20.04_build.txt or 20.04_base.txt exists @@ -120,17 +124,12 @@ fi # if the windows introspection library is not installed, clone and install if [[ !$(dpkg -l | grep -q libosi) ]]; then pushd /tmp - curl -LJO https://github.com/panda-re/libosi/releases/download/v${LIBOSI_VERSION}/libosi_${UBUNTU_VERSION}.deb + curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_${UBUNTU_VERSION}.deb $SUDO dpkg -i /tmp/libosi_${UBUNTU_VERSION}.deb rm -rf /tmp/libosi_${UBUNTU_VERSION}.deb popd fi -# PyPANDA needs CFFI from pip (the version in apt is too old) -# Install system-wide since PyPANDA install will also be system-wide -$SUDO python3 -m pip install pip -$SUDO python3 -m pip install "cffi>1.14.3" - progress "Trying to update DTC submodule" git submodule update --init dtc || true @@ -139,6 +138,10 @@ if [ -d "build" ]; then rm -rf "build" fi +# PyPANDA needs CFFI from pip (the version in apt is too old) +# Install system-wide since PyPANDA install will also be system-wide +$SUDO pip install -r ./panda/python/core/requirements.txt + progress "Building PANDA..." mkdir build pushd build