Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use vcpkg to build deb, apk and macOS packages #33

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 8 additions & 11 deletions .github/workflows/ci-build-binary-artifacts.yaml
Original file line number Diff line number Diff line change
@@ -55,9 +55,6 @@ jobs:
- name: Package Pulsar source
run: build-support/generate-source-archive.sh

- uses: docker/setup-buildx-action@v2
- run: build-support/copy-deps-versionfile.sh

- name: Build dependencies Docker image
uses: docker/build-push-action@v3
with:
@@ -66,8 +63,6 @@ jobs:
tags: build:latest
platforms: linux/${{matrix.cpu.platform}}
build-args: PLATFORM=${{matrix.cpu.arch}}
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Build packages
run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest
@@ -208,17 +203,19 @@ jobs:
steps:
- name: checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: recursive

- name: Install dependencies
run: |
export ARCH=${{ matrix.arch }}
./pkg/mac/build-static-library.sh
run: ARCH=${{ matrix.arch }} ./pkg/mac/build-static-library.sh

- name: Zip artifact
run: |
cd ./pkg/mac/.install
zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/*
cp macos-${{ matrix.arch }}.zip ../../../
mv build/include/pulsar/Version.h include/pulsar/
mv build/lib/*.a lib/
mv build/lib/*.dylib lib/
zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/*.a ./lib/*.dylib

- name: Upload artifacts
uses: actions/upload-artifact@v3
304 changes: 0 additions & 304 deletions .github/workflows/ci-pr-validation.yaml
Original file line number Diff line number Diff line change
@@ -28,256 +28,15 @@ concurrency:
cancel-in-progress: true

jobs:

formatting-check:
name: Formatting Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Run clang-format style check for C/C++/Protobuf programs.
uses: jidicula/clang-format-action@v4.11.0
with:
clang-format-version: '11'
exclude-regex: '.*\.(proto|hpp)'

wireshark-dissector-build:
name: Build the Wireshark dissector
needs: formatting-check
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
matrix:
os: [ubuntu-20.04, macos-12]

steps:
- name: checkout
uses: actions/checkout@v3

- name: Install deps (Ubuntu)
if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: |
sudo apt-get update -y
sudo apt-get install -y protobuf-compiler libprotobuf-dev wireshark-dev

- name: Install deps (macOS)
if: ${{ startsWith(matrix.os, 'macos') }}
run: |
# See https://github.com/Homebrew/homebrew-core/issues/157142
export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
cd /usr/local/bin
rm -f 2to3* idle3* pydoc* python3*
rm -f /usr/local/share/man/man1/python3.1 /usr/local/lib/pkgconfig/python3*
cd /usr/local/Frameworks/Python.framework
rm -rf Headers Python Resources Versions/Current
brew update
brew install pkg-config wireshark protobuf
- name: Build wireshark plugin
run: |
cmake -S wireshark -B build-wireshark
cmake --build build-wireshark

unit-tests:
name: Run unit tests
needs: formatting-check
runs-on: ubuntu-22.04
timeout-minutes: 120

steps:
- name: checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: recursive

- name: Build core libraries
run: |
cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF
cmake --build . -j8

- name: Check formatting
run: |
./vcpkg/vcpkg format-manifest vcpkg.json
if [[ $(git diff | wc -l) -gt 0 ]]; then
echo "Please run `./vcpkg/vcpkg format-manifest vcpkg.json` to reformat vcpkg.json"
exit 1
fi

- name: Build tests
run: |
cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON
cmake --build . -j8

- name: Install gtest-parallel
run: |
sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py

- name: Run unit tests
run: RETRY_FAILED=3 ./run-unit-tests.sh

- name: Build perf tools
run: |
cmake . -DINTEGRATE_VCPKG=ON -DBUILD_TESTS=ON -DBUILD_PERF_TOOLS=ON
cmake --build . -j8

- name: Verify custom vcpkg installation
run: |
mv vcpkg /tmp/
cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_TOOLCHAIN_FILE="/tmp/vcpkg/scripts/buildsystems/vcpkg.cmake"

cpp20-build:
name: Build with the C++20 standard
needs: formatting-check
runs-on: ubuntu-22.04
timeout-minutes: 60

steps:
- name: checkout
uses: actions/checkout@v3
- name: Install deps
run: |
sudo apt-get update -y
sudo apt-get install -y libcurl4-openssl-dev libssl-dev \
protobuf-compiler libprotobuf-dev libboost-dev \
libboost-dev libboost-program-options-dev \
libzstd-dev libsnappy-dev libgmock-dev libgtest-dev
- name: CMake
run: cmake -B build -DBUILD_PERF_TOOLS=ON -DCMAKE_CXX_STANDARD=20
- name: Build
run: |
cmake --build build -j8 --target pulsarShared pulsarStatic
cmake --build build -j8

cpp-build-windows:
timeout-minutes: 120
name: Build CPP Client on ${{ matrix.name }}
needs: unit-tests
runs-on: ${{ matrix.os }}
env:
VCPKG_ROOT: '${{ github.workspace }}/vcpkg'
INSTALL_DIR: 'C:\\pulsar-cpp'
strategy:
fail-fast: false
matrix:
include:
- name: 'Windows x64'
os: windows-2019
triplet: x64-windows-static
suffix: 'windows-win64'
generator: 'Visual Studio 16 2019'
arch: '-A x64'
- name: 'Windows x86'
os: windows-2019
triplet: x86-windows-static
suffix: 'windows-win32'
generator: 'Visual Studio 16 2019'
arch: '-A Win32'

steps:
- name: checkout
uses: actions/checkout@v3

- name: Restore vcpkg and its artifacts.
uses: actions/cache@v3
id: vcpkg-cache
with:
path: |
${{ env.VCPKG_ROOT }}
vcpkg_installed
!${{ env.VCPKG_ROOT }}/.git
!${{ env.VCPKG_ROOT }}/buildtrees
!${{ env.VCPKG_ROOT }}/packages
!${{ env.VCPKG_ROOT }}/downloads
key: |
${{ runner.os }}-${{ matrix.triplet}}-${{ hashFiles( 'vcpkg.json' ) }}

- name: Get vcpkg(windows)
if: ${{ runner.os == 'Windows' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
run: |
cd ${{ github.workspace }}
mkdir build -force
git clone https://github.com/Microsoft/vcpkg.git
cd vcpkg
.\bootstrap-vcpkg.bat

- name: remove system vcpkg(windows)
if: runner.os == 'Windows'
run: rm -rf "$VCPKG_INSTALLATION_ROOT"
shell: bash

- name: Install vcpkg packages
run: |
${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }}

- name: Configure
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
cmake \
-B ./build-1 \
-G "${{ matrix.generator }}" ${{ matrix.arch }} \
-DUSE_ASIO=ON \
-DBUILD_TESTS=OFF \
-DVCPKG_TRIPLET="${{ matrix.triplet }}" \
-DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \
-S .
fi

- name: Install
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
cmake --build ./build-1 --parallel --config Release
cmake --install ./build-1
fi

- name: Test examples
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
cd win-examples
cmake \
-G "${{ matrix.generator }}" ${{ matrix.arch }} \
-DLINK_STATIC=OFF \
-DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \
-B build-dynamic
cmake --build build-dynamic --config Release
cmake \
-G "${{ matrix.generator }}" ${{ matrix.arch }} \
-DLINK_STATIC=ON \
-DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \
-B build-static
cmake --build build-static --config Release
./build-static/Release/win-example.exe
fi

- name: Build (Debug)
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
cmake \
-B ./build-2 \
-G "${{ matrix.generator }}" ${{ matrix.arch }} \
-DUSE_ASIO=ON \
-DBUILD_TESTS=OFF \
-DVCPKG_TRIPLET="${{ matrix.triplet }}" \
-DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \
-DCMAKE_BUILD_TYPE=Debug \
-S .
cmake --build ./build-2 --parallel --config Debug
fi

package:
name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}}
runs-on: ubuntu-22.04
needs: unit-tests
timeout-minutes: 500

strategy:
fail-fast: true
matrix:
pkg:
- { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' }
- { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' }
- { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' }
cpu:
- { arch: 'x86_64', platform: 'x86_64' }
@@ -292,9 +51,6 @@ jobs:
- name: Package Pulsar source
run: build-support/generate-source-archive.sh

- uses: docker/setup-buildx-action@v2
- run: build-support/copy-deps-versionfile.sh

- name: Build dependencies Docker image
uses: docker/build-push-action@v3
with:
@@ -303,66 +59,6 @@ jobs:
tags: build:latest
platforms: linux/${{matrix.cpu.platform}}
build-args: PLATFORM=${{matrix.cpu.arch}}
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Build packages
run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest

cpp-build-macos:
timeout-minutes: 120
name: Build CPP Client on macOS
needs: formatting-check
runs-on: macos-12
steps:
- name: checkout
uses: actions/checkout@v3

- name: Install dependencies
run: brew install openssl protobuf boost zstd snappy googletest

- name: Configure (default)
shell: bash
run: cmake -B ./build-macos -S .

- name: Compile
shell: bash
run: |
cmake --build ./build-macos --parallel --config Release

- name: Build with C++20
shell: bash
run: |
cmake -B build-macos-cpp20 -DCMAKE_CXX_STANDARD=20
cmake --build build-macos-cpp20 -j8

cpp-build-macos-static:
timeout-minutes: 120
name: Build CPP Client on macOS with static dependencies
runs-on: macos-12
needs: unit-tests
steps:
- name: checkout
uses: actions/checkout@v3

- name: Build libraries
run: ./pkg/mac/build-static-library.sh

- name: Test static libraries
run: |
export PULSAR_DIR=$PWD/pkg/mac/.install
echo "Build with static library"
clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a
./static.out
echo "Build with dynamic library"
clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar
./dynamic.out

# Job that will be required to complete and depends on all the other jobs
check-completion:
name: Check Completion
runs-on: ubuntu-latest
needs: [formatting-check, wireshark-dissector-build, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos]

steps:
- run: true
73 changes: 0 additions & 73 deletions .github/workflows/codeql-analysis.yml

This file was deleted.

29 changes: 0 additions & 29 deletions build-support/copy-deps-versionfile.sh

This file was deleted.

24 changes: 0 additions & 24 deletions build-support/dep-version.py

This file was deleted.

2 changes: 1 addition & 1 deletion build-support/merge_archives_vcpkg.sh
Original file line number Diff line number Diff line change
@@ -29,4 +29,4 @@ fi
CMAKE_BUILD_DIRECTORY=$1
./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
$CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
$(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug)
$(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug | grep "$TRIPLET")
28 changes: 0 additions & 28 deletions dependencies.yaml

This file was deleted.

32 changes: 20 additions & 12 deletions pkg/apk/APKBUILD
Original file line number Diff line number Diff line change
@@ -38,19 +38,27 @@ build() {
if [ "$CBUILD" != "$CHOST" ]; then
CMAKE_CROSSOPTS="-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_HOST_SYSTEM_NAME=Linux"
fi
cmake \
-DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_INSTALL_LIBDIR=lib \
-DBUILD_SHARED_LIBS=True \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_FLAGS="$CXXFLAGS" \
-DCMAKE_C_FLAGS="$CFLAGS" \
-DBUILD_TESTS=OFF \
-DLINK_STATIC=ON \
${CMAKE_CROSSOPTS} ${ROOT_DIR}
make -j6
git clone https://github.com/microsoft/vcpkg.git
mv vcpkg $ROOT_DIR/
if [[ $PLATFORM == "aarch64" ]]; then
export VCPKG_FORCE_SYSTEM_BINARIES=1
fi
set +e
cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_MAKE_PROGRAM=make \
-DCMAKE_INSTALL_PREFIX="$pkgdir" \
-DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON \
$ROOT_DIR
if [[ $? -ne 0 ]]; then
cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-CMakeCache.txt.log
cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-out.log
cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-err.log
exit 1
fi
set -e
cmake --build build -j8
}

package() {
make DESTDIR="$pkgdir" install
cmake --build build --target install
}
98 changes: 15 additions & 83 deletions pkg/apk/Dockerfile
Original file line number Diff line number Diff line change
@@ -21,91 +21,23 @@ FROM alpine:3.12

ARG PLATFORM

# perl is required to install OpenSSL
RUN apk add \
build-base \
RUN apk add build-base \
linux-headers \
abuild \
bash \
curl \
g++ \
make \
curl \
cmake \
python3 \
py3-pip \
git \
ninja \
zip \
unzip \
openssl-dev \
perl \
sudo

RUN pip3 install pyyaml

ADD .build/dependencies.yaml /
ADD .build/dep-version.py /usr/local/bin

# Download and compile boost
RUN BOOST_VERSION=$(dep-version.py boost) && \
curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \
tar zxf boost-${BOOST_VERSION}.tar.gz && \
cd boost-${BOOST_VERSION} && \
./bootstrap.sh --with-libraries=regex && \
./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION}

# Download and compile protobuf
RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
cd protobuf-${PROTOBUF_VERSION}/ && \
CXXFLAGS=-fPIC ./configure && \
make -j8 && make install && \
rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}

# ZLib
RUN ZLIB_VERSION=$(dep-version.py zlib) && \
curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
tar xfz v${ZLIB_VERSION}.tar.gz && \
cd zlib-${ZLIB_VERSION} && \
CFLAGS="-fPIC -O3" ./configure && \
make -j8 && make install && \
rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}

# Zstandard
RUN ZSTD_VERSION=$(dep-version.py zstd) && \
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
cd zstd-${ZSTD_VERSION} && \
CFLAGS="-fPIC -O3" make -j8 && \
make install && \
rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz

# Snappy
RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
tar xfz ${SNAPPY_VERSION}.tar.gz && \
cd snappy-${SNAPPY_VERSION} && \
CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
make -j8 && make install && \
rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz

RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \
make -j8 && make install && \
rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}

ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:
ENV OPENSSL_ROOT_DIR /usr/local/ssl/

# LibCurl
RUN CURL_VERSION=$(dep-version.py curl) && \
CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
tar xfz curl-${CURL_VERSION}.tar.gz && \
cd curl-${CURL_VERSION} && \
CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \
make -j8 && make install && \
rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}


bash

# Build latest CMake for vcpkg
RUN curl -O -L https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0.tar.gz \
&& tar zxf cmake-3.30.0.tar.gz \
&& cd cmake-3.30.0 \
&& ./configure && make -j8 && make install \
&& cd .. && rm -rf cmake-3.30.0 cmake-3.30.0.tar.gz
2 changes: 1 addition & 1 deletion pkg/apk/docker-build-apk-arm64.sh
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
IMAGE_NAME=${1:-apachepulsar/pulsar-build:alpine-3.16-arm64}

docker run -v $ROOT_DIR:/pulsar-client-cpp \
--env PLATFORM=arm64 \
--env PLATFORM=aarch64 \
$IMAGE_NAME \
/pulsar-client-cpp/pkg/apk/build-apk.sh

90 changes: 6 additions & 84 deletions pkg/deb/Dockerfile
Original file line number Diff line number Diff line change
@@ -29,88 +29,10 @@ RUN apt-get update -y && \
curl \
g++ \
make \
perl \
dpkg-dev \
python3 \
python3-pip

RUN pip3 install pyyaml

ADD .build/dependencies.yaml /
ADD .build/dep-version.py /usr/local/bin

# Download and compile boost
RUN BOOST_VERSION=$(dep-version.py boost) && \
curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \
tar zxf boost-${BOOST_VERSION}.tar.gz && \
cd boost-${BOOST_VERSION} && \
./bootstrap.sh --with-libraries=regex && \
./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION}

RUN CMAKE_VERSION=$(dep-version.py cmake) && \
curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \
tar xfz cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \
cp cmake-${CMAKE_VERSION}-linux-${PLATFORM}/bin/* /usr/bin/ && \
cp -r cmake-${CMAKE_VERSION}-linux-${PLATFORM}/share/cmake-* /usr/share/ && \
rm -rf cmake-${CMAKE_VERSION}-linux-${PLATFORM} cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz

# Download and compile protobuf
RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
cd protobuf-${PROTOBUF_VERSION}/ && \
CXXFLAGS=-fPIC ./configure && \
make -j8 && make install && ldconfig && \
rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}

# ZLib
RUN ZLIB_VERSION=$(dep-version.py zlib) && \
curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
tar xfz v${ZLIB_VERSION}.tar.gz && \
cd zlib-${ZLIB_VERSION} && \
CFLAGS="-fPIC -O3" ./configure && \
make -j8 && make install && \
rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}

# Zstandard
RUN ZSTD_VERSION=$(dep-version.py zstd) && \
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
cd zstd-${ZSTD_VERSION} && \
CFLAGS="-fPIC -O3" make -j8 && \
make install && \
rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz

# Snappy
RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
tar xfz ${SNAPPY_VERSION}.tar.gz && \
cd snappy-${SNAPPY_VERSION} && \
CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
make -j8 && make install && \
rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz

RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \
make -j8 && make install && \
rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}

ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:
ENV OPENSSL_ROOT_DIR /usr/local/ssl/

# LibCurl
RUN CURL_VERSION=$(dep-version.py curl) && \
CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
tar xfz curl-${CURL_VERSION}.tar.gz && \
cd curl-${CURL_VERSION} && \
CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \
make -j8 && make install && \
rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}


zip \
unzip \
tar \
git \
pkg-config \
ninja-build
27 changes: 21 additions & 6 deletions pkg/deb/build-deb.sh
Original file line number Diff line number Diff line change
@@ -20,6 +20,10 @@

set -e -x

if [[ $# -gt 0 ]]; then
CMAKE_ARCH=$1
fi

cd /pulsar-client-cpp
SRC_ROOT_DIR=$(pwd)
cd pkg/deb
@@ -36,13 +40,24 @@ mkdir BUILD
cd BUILD
tar xfz $SRC_ROOT_DIR/apache-pulsar-client-cpp-$POM_VERSION.tar.gz
pushd $CPP_DIR

# link libraries for protoc
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH

git clone https://github.com/microsoft/vcpkg.git
if [[ $CMAKE_ARCH ]]; then
curl -O -L https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-$CMAKE_ARCH.tar.gz
export PATH=$PWD/cmake-3.30.0-linux-$CMAKE_ARCH/bin/:$PATH
tar zxf cmake-*.tar.gz
fi
chmod +x $(find . -name "*.sh")
cmake . -DBUILD_TESTS=OFF -DBUILD_PERF_TOOLS=OFF -DLINK_STATIC=ON
make -j 3
if [[ $CMAKE_ARCH == "aarch64" ]]; then
export VCPKG_FORCE_SYSTEM_BINARIES=1
fi
cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON
cmake --build build -j8
./build-support/merge_archives_vcpkg.sh $PWD/build

cp build/lib/libpulsar.a lib/libpulsar.a
cp build/lib/libpulsar.so lib/libpulsar.so
cp build/libpulsarwithdeps.a lib/libpulsarwithdeps.a
popd

DEST_DIR=apache-pulsar-client
4 changes: 3 additions & 1 deletion pkg/deb/docker-build-deb-arm64.sh
Original file line number Diff line number Diff line change
@@ -27,4 +27,6 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-arm64}
docker run -v $ROOT_DIR:/pulsar-client-cpp \
--env PLATFORM=arm64 \
$IMAGE_NAME \
/pulsar-client-cpp/pkg/deb/build-deb.sh
/pulsar-client-cpp/pkg/deb/build-deb.sh \
aarch64
https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-aarch64.tar.gz
3 changes: 2 additions & 1 deletion pkg/deb/docker-build-deb-x86_64.sh
Original file line number Diff line number Diff line change
@@ -27,4 +27,5 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-x86_64}
docker run -v $ROOT_DIR:/pulsar-client-cpp \
--env PLATFORM=amd64 \
$IMAGE_NAME \
/pulsar-client-cpp/pkg/deb/build-deb.sh
/pulsar-client-cpp/pkg/deb/build-deb.sh \
x86_64
190 changes: 16 additions & 174 deletions pkg/mac/build-static-library.sh
Original file line number Diff line number Diff line change
@@ -19,179 +19,21 @@
#

set -ex
cd `dirname $0`
cd `dirname $0`/../..

pip3 install pyyaml

MACOSX_DEPLOYMENT_TARGET=10.15
if [[ -z ${ARCH} ]]; then
ARCH=`uname -m`
fi

BUILD_DIR=$PWD/.build
INSTALL_DIR=$PWD/.install
PREFIX=$BUILD_DIR/install
mkdir -p $BUILD_DIR
cp -f ../../build-support/dep-version.py $BUILD_DIR/
cp -f ../../dependencies.yaml $BUILD_DIR/

pushd $BUILD_DIR

BOOST_VERSION=$(./dep-version.py boost)
ZLIB_VERSION=$(./dep-version.py zlib)
OPENSSL_VERSION=$(./dep-version.py openssl)
PROTOBUF_VERSION=$(./dep-version.py protobuf)
ZSTD_VERSION=$(./dep-version.py zstd)
SNAPPY_VERSION=$(./dep-version.py snappy)
CURL_VERSION=$(./dep-version.py curl)

if [ ! -f boost/.done ]; then
echo "Building Boost $BOOST_VERSION"
curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz
tar zxf boost-${BOOST_VERSION}.tar.gz
mkdir -p $PREFIX/include
pushd boost-${BOOST_VERSION}
./bootstrap.sh
./b2 headers
cp -rf boost $PREFIX/include/
popd
mkdir -p boost
touch boost/.done
else
echo "Using cached Boost $BOOST_VERSION"
fi

if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then
echo "Building ZLib $ZLIB_VERSION"
curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz
tar zxf zlib-${ZLIB_VERSION}.tar.gz
pushd zlib-$ZLIB_VERSION
CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
make -j16
make install
touch .done
popd
else
echo "Using cached ZLib $ZLIB_VERSION"
fi

OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then
echo "Building OpenSSL $OPENSSL_VERSION"
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz
tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz

pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
if [[ $ARCH = 'arm64' ]]; then
PLATFORM=darwin64-arm64-cc
else
PLATFORM=darwin64-x86_64-cc
fi
CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM
make -j8 >/dev/null
make install_sw >/dev/null
popd

touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done
else
echo "Using cached OpenSSL $OPENSSL_VERSION"
fi

if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then
echo "Building Protobuf $PROTOBUF_VERSION"
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
pushd protobuf-${PROTOBUF_VERSION}
pushd cmake/
# Build protoc that can run on both x86 and arm architectures
cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch x86_64 -arch arm64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
-Dprotobuf_BUILD_TESTS=OFF \
-DCMAKE_INSTALL_PREFIX=$PREFIX
cmake --build build -j16 --target install
popd

# Retain the library for one architecture so that `ar` can work on the library
pushd $PREFIX/lib
mv libprotobuf.a libprotobuf_universal.a
lipo libprotobuf_universal.a -thin ${ARCH} -output libprotobuf.a
popd
touch .done
popd
else
echo "Using cached Protobuf $PROTOBUF_VERSION"
fi

if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then
echo "Building ZStd $ZSTD_VERSION"
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
tar zxf zstd-${ZSTD_VERSION}.tar.gz
pushd zstd-${ZSTD_VERSION}
CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \
make -j16 -C lib install-static install-includes
touch .done
popd
if [[ $ARCH == "arm64" ]]; then
VCPKG_TARGET_TRIPLET=arm64-osx
CMAKE_OSX_ARCHITECTURES=arm64
else
echo "Using cached ZStd $ZSTD_VERSION"
fi

if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
echo "Building Snappy $SNAPPY_VERSION"
curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz
tar zxf ${SNAPPY_VERSION}.tar.gz
pushd snappy-${SNAPPY_VERSION}
# Without this patch, snappy 1.10 will report a sign-compare error, which cannot be suppressed with the -Wno-sign-compare option in CI
curl -O -L https://raw.githubusercontent.com/microsoft/vcpkg/2024.02.14/ports/snappy/no-werror.patch
patch <no-werror.patch
CXXFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF
make -j16
make install
touch .done
popd
else
echo "Using cached Snappy $SNAPPY_VERSION"
fi

if [ ! -f curl-${CURL_VERSION}/.done ]; then
echo "Building LibCurl $CURL_VERSION"
CURL_VERSION_=${CURL_VERSION//./_}
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz
tar zxf curl-${CURL_VERSION}.tar.gz
pushd curl-${CURL_VERSION}
# Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl.
cp -rf $PREFIX/include/openssl include/
CFLAGS="-I$PREFIX/include -fPIC -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
./configure --with-ssl=$PREFIX \
--without-nghttp2 \
--without-libidn2 \
--disable-ldap \
--without-brotli \
--without-secure-transport \
--without-librtmp \
--disable-ipv6 \
--without-libpsl \
--host=$ARCH-apple-darwin \
--prefix=$PREFIX
make -j16 install
touch .done
popd
else
echo "Using cached LibCurl $CURL_VERSION"
fi

popd # pkg/mac
cd ../../ # project root

cmake -B build-static -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
-DLINK_STATIC=ON \
-DBUILD_TESTS=OFF \
-DBUILD_DYNAMIC_LIB=ON \
-DBUILD_STATIC_LIB=ON \
-DCMAKE_OSX_ARCHITECTURES=${ARCH} \
-DCMAKE_PREFIX_PATH=$PREFIX \
-DOPENSSL_ROOT_DIR=$PREFIX \
-DPROTOC_PATH=$PREFIX/bin/protoc \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
-DCMAKE_BUILD_TYPE=Release
cmake --build build-static -j16 --target install
VCPKG_TARGET_TRIPLET=x64-osx
CMAKE_OSX_ARCHITECTURES=x86_64
fi

cmake -B build \
-DCMAKE_BUILD_TYPE=Release \
-DINTEGRATE_VCPKG=ON -DBUILD_TESTS=OFF -DBUILD_PERF_TOOLS=OFF \
-DCMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \
-DVCPKG_TARGET_TRIPLET=$VCPKG_TARGET_TRIPLET
cmake --build build -j8
TRIPLET=$VCPKG_TARGET_TRIPLET ./build-support/merge_archives_vcpkg.sh $PWD/build
mv build/libpulsarwithdeps.a build/lib/
9 changes: 1 addition & 8 deletions pkg/rpm/Dockerfile
Original file line number Diff line number Diff line change
@@ -30,16 +30,9 @@ RUN yum update -y && \
rpm-build \
which \
createrepo \
git \
python3 \
python3-pip
git
RUN dnf --enablerepo=powertools install -y libstdc++-static

RUN pip3 install pyyaml

ADD .build/dependencies.yaml /
ADD .build/dep-version.py /usr/local/bin

# Vcpkg does not provide pre-built binaries for Arm architectures so we need to build vcpkg from source
RUN yum install -y cmake
RUN dnf --enablerepo=devel install -y ninja-build