Merge pull request #272 from codeadict/osx_fix

Depend on system snappy (With OSX fix)

.github/workflows/erlang.yml

@@ -2,36 +2,74 @@ name: Erlang CI
 
 on:
   push:
-    branches: [ develop ]
+    branches: [develop]
   pull_request:
-    branches: [ develop ]
+    branches: [develop]
 
-jobs:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
+jobs:
   build:
-
-    runs-on: ubuntu-latest
+    name: Test on ${{ matrix.os }} with OTP ${{ matrix.otp }}
+    runs-on: ${{ matrix.os }}
 
     strategy:
       fail-fast: false
       matrix:
-        otp:
-          - "25"
-          - "24"
-          - "22"
-
-    container:
-      image: erlang:${{ matrix.otp }}
+        otp: [22, 23, 24, 25]
+        os: [ubuntu-latest, macos-latest]
+        # OTP lower than 23 does not run on ubuntu-latest (22.04), see
+        # https://github.com/erlef/setup-beam#compatibility-between-operating-system-and-erlangotp
+        exclude:
+          - otp: 22
+            os: ubuntu-latest
+          - otp: 23
+            os: ubuntu-latest
+        include:
+          - otp: 22
+            os: ubuntu-20.04
+          - otp: 23
+            os: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Cache rebar3
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/rebar3
+          key: ${{runner.os}}-rebar3-cache-${{matrix.otp}}-v2-${{ hashFiles(format('rebar.lock')) }}
+      - name: Install dependencies (Linux)
+        if: ${{ startsWith(matrix.os, 'ubuntu') }}
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get -qq install libsnappy-dev libc6-dev
+      - name: Configure Homebrew cache
+        if: ${{ startsWith(matrix.os, 'macos') }}
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/Homebrew/
+            ~/Library/Caches/Homebrew/downloads/
+          key: brew-${{ runner.os }}-${{ matrix.otp  }}
+      - name: Install Dependencies (OSX)
+        if: ${{ startsWith(matrix.os, 'macos') }}
+        run: |
+          export majorversion="$(cut -d '.' -f 1 <<< "${{ matrix.otp }}")"
+          brew install coreutils erlang@$majorversion snappy
+          echo "/usr/local/opt/erlang@$majorversion/bin" >> $GITHUB_PATH
+      - name: Install Erlang/OTP
+        # setup beam doesn't provide MacOS packages
+        # we use Homebrew to instal them
+        if: ${{ !startsWith(matrix.os , 'macos') }}
+        uses: erlef/setup-beam@v1
+        with:
+          otp-version: ${{ matrix.otp }}
       - name: Compile
-        run: apt-get update && apt-get install -y cmake && make
-      - name: Run tests
-        run: apt-get update && apt-get install -y cmake libc6-dev && make check
-      - name: rebar3 compile 
         run: ./rebar3 compile
       - name: Run xref and dialyzer
         run: ./rebar3 do xref, dialyzer
-      - name: Run eunit
+      - name: Run tests
         run: ./rebar3 as gha do eunit

Makefile

@@ -16,7 +16,6 @@ cover: test
 	$(REBAR) cover
 
 test: compile
-	$(MAKE) -C c_src test
 	$(REBAR) as test do eunit
 
 dialyzer:

c_src/Makefile

@@ -1,52 +1,57 @@
 LEVELDB_VSN ?= "2.0.38"
-SNAPPY_VSN ?= "1.1.9"
-BASEDIR = $(shell pwd)
+C_SRC_DIR := $(shell pwd)
+PROJECT_DIR := $(abspath $(C_SRC_DIR)/..)
+TARGET_DIR := $(PROJECT_DIR)/priv/
 
-LDFLAGS := $(LDFLAGS) -L$(BASEDIR)/system/lib
-LD_LIBRARY_PATH := $(BASEDIR)/system/lib:$(LD_LIBRARY_PATH)
-CFLAGS := $(CFLAGS) -I $(BASEDIR)/system/include -I. -I $(BASEDIR)/leveldb/include -fPIC
-CXXFLAGS := $(CXXFLAGS) -I $(BASEDIR)/system/include -I. -I $(BASEDIR)/leveldb/include -fPIC
+# LevelDB tools
+TOOLS = \
+	leveldb_repair \
+	perf_dump \
+	sst_rewrite \
+	sst_scan
+
+LDFLAGS := $(LDFLAGS) -lsnappy -lpthread
+CFLAGS := $(CFLAGS) -I. -I $(C_SRC_DIRC_SRC_DIR)/leveldb/include -fPIC
+CXXFLAGS := $(CXXFLAGS) -I. -I $(C_SRC_DIR)/leveldb/include -fPIC
+TEST_CXXFLAGS := $(CXXFLAGS) -Wno-narrowing
+
+ifeq ($(shell uname -s), Darwin)
+
+	# OSX with homebrew
+  HAS_BREW := $(shell command -v brew;)
+
+	ifdef HAS_BREW
+		SNAPPY_DIR ?= $(shell brew --prefix snappy)
+	endif
+
+	# Default dir (Mac ports)
+	SNAPPY_DIR ?= /usr/local/opt/snappy
+
+  LDFLAGS += -L${SNAPPY_DIR}/lib
+
+	# Resolves C++11 narrowing error on Mac OS
+	TEST_CXXFLAGS += -Wno-c++11-narrowing
+endif
 
 get-deps:
 	git config --global --add safe.directory /__w/eleveldb/eleveldb
 	echo "ubuntu-latest image with otp-22, are you happy now?"
-	if [ ! -r snappy-$(SNAPPY_VSN).tar.gz ]; then \
-	    wget -O snappy-$(SNAPPY_VSN).tar.gz https://github.com/google/snappy/archive/refs/tags/$(SNAPPY_VSN).tar.gz; \
-	fi
 	if [ ! -d leveldb ]; then \
-	    git clone https://github.com/basho/leveldb && \
-	    (cd leveldb && git checkout $(LEVELDB_VSN)) && \
-	    (cd leveldb && git submodule update --init); \
+	    git clone --depth=1 --branch=$(LEVELDB_VSN) https://github.com/basho/leveldb && \
+	    (cd leveldb && git submodule update --depth 1 --init); \
 	fi
 
-compile: get-deps snappy ldb
-	cp leveldb/perf_dump leveldb/sst_rewrite leveldb/sst_scan leveldb/leveldb_repair ../priv
+compile: get-deps ldb
+	for tool in $(TOOLS); do cp leveldb/$$tool $(TARGET_DIR); done
 
 ldb:
-	$(MAKE) LDFLAGS="$(LDFLAGS) -lsnappy -lpthread" LD_LIBRARY_PATH="$(LD_LIBRARY_PATH)" -C leveldb all
-	$(MAKE) LDFLAGS="$(LDFLAGS) -lsnappy -lpthread" LD_LIBRARY_PATH="$(LD_LIBRARY_PATH)" -C leveldb tools
-
-snappy: system/lib/libsnappy.a
-
-system/lib/libsnappy.a:
-	tar -xzf snappy-$(SNAPPY_VSN).tar.gz && \
-	(cd snappy-$(SNAPPY_VSN) && \
-	 git submodule update --init && \
-	 if [ -r autogen.sh ]; then \
-	   ./autogen.sh && ./configure --prefix=$(BASEDIR)/system && $(MAKE) && $(MAKE) install; \
-	 else \
-	   mkdir build && cd build && \
-	   mkdir -p $(BASEDIR)/system && \
-	   cmake -D SNAPPY_BUILD_TESTS=0 -D SNAPPY_BUILD_BENCHMARKS=0 \
-	         -D CMAKE_INSTALL_PREFIX=$(BASEDIR)/system \
-	     ..; \
-	  fi && \
-	  $(MAKE) && $(MAKE) install)
-	mv system/lib64 system/lib || true
+	@echo "Building LevelDB..."
+	$(MAKE) LDFLAGS="" -C leveldb all
+	$(MAKE) LDFLAGS="$(LDFLAGS)" -C leveldb tools
 
 clean:
 	$(MAKE) -C leveldb clean
-	rm -rf system snappy-$(SNAPPY_VSN)/build
+	for tool in $(TOOLS); do rm -f $(TARGET_DIR)$$tool; done
 
 test: compile
-	$(MAKE) CXXFLAGS="$(CXXFLAGS) -Wno-narrowing" LDFLAGS="$(LDFLAGS) -lsnappy -lpthread" LD_LIBRARY_PATH="$(LD_LIBRARY_PATH)" -C leveldb test
+	$(MAKE) LDFLAGS="$(LDFLAGS)" CXXFLAGS="$(TEST_CXXFLAGS)" CFLAGS="$(CFLAGS)" -C leveldb test

rebar.config

@@ -2,41 +2,50 @@
 {minimum_otp_vsn, "22.0"}.
 
 {eunit_opts, [verbose]}.
-{so_name, "eleveldb.so"}.
 
 {plugins, [{eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}}, pc]}.
 
-{provider_hooks,
- [
-  {post,
-   [
-    {compile, {pc, compile}},
-    {clean, {pc, clean}}
-   ]
-  }
- ]
-}.
-
-{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used,
-    deprecated_function_calls, deprecated_functions]}.
+{provider_hooks, [
+    {post, [
+        {compile, {pc, compile}},
+        {clean, {pc, clean}}
+    ]}
+]}.
 
-{port_sources, ["c_src/*.cc"]}.
+{xref_checks, [
+    undefined_function_calls,
+    undefined_functions,
+    locals_not_used,
+    deprecated_function_calls,
+    deprecated_functions
+]}.
 
 {erl_opts, [warnings_as_errors, debug_info]}.
 
 {profiles, [
     {gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]}
 ]}.
 
+{port_specs, [{"priv/eleveldb.so", ["c_src/*.cc"]}]}.
+
+{artifacts, ["priv/eleveldb.so"]}.
+
 {port_env, [
-         %% Make sure to set -fPIC when compiling leveldb
-             {"CFLAGS", "$CFLAGS -Wall -O3 -fPIC"},
-             {"CXXFLAGS", "$CXXFLAGS -Wall -O3 -fPIC"},
-             {"DRV_CFLAGS", "$DRV_CFLAGS -O3 -Wall -I c_src/leveldb/include -I c_src/leveldb -I c_src/system/include"},
-             {"DRV_LDFLAGS", "$DRV_LDFLAGS c_src/leveldb/libleveldb.a c_src/system/lib/libsnappy.a -lstdc++"}
-             ]}.
+    %% Make sure to set -fPIC when compiling leveldb
+    {"CXXFLAGS", "$CXXFLAGS -Wall -O3 -fPIC -I c_src/leveldb/include -I c_src/leveldb"},
+    {"LDFLAGS", "$LDFLAGS c_src/leveldb/libleveldb.a -lsnappy -lstdc++"}
+]}.
 
-{pre_hooks, [{'get-deps', "./make -C c_src get-deps"},
-             {compile, "./make -C c_src compile"}]}.
+{pre_hooks, [
+    {"(linux|darwin|solaris)", 'get-deps', "./make -C c_src get-deps"},
+    {"(freebsd|netbsd|openbsd)", 'get-deps', "gmake -C c_src get-deps"},
+    {"(linux|darwin|solaris)", compile, "make -C c_src compile"},
+    {"(freebsd|netbsd|openbsd)", compile, "gmake -C c_src compile"},
+    {"(linux|darwin|solaris)", eunit, "make -C c_src test"},
+    {"(freebsd|netbsd|openbsd)", eunit, "gmake -C c_src test"}
+]}.
 
-{post_hooks, [{clean, "./make -C c_src clean"}]}.
+{post_hooks, [
+    {"(linux|darwin|solaris)", clean, "make -C c_src clean"},
+    {"(freebsd|netbsd|openbsd)", clean, "gmake -C c_src clean"}
+]}.

rebar.config.script

@@ -1,30 +1,42 @@
 %%-*- mode: erlang -*-
 
+AddPortEnv = fun(Config, Flags, NewValue) ->
+    {port_env, Vals} = lists:keyfind(port_env, 1, Config),
+    Fold = fun({Flag, Val}, Acc) ->
+        case lists:member(Flag, Flags) of
+            true ->
+                [{Flag, Val ++ NewValue} | Acc];
+            false ->
+                Acc
+        end
+    end,
+    NewVals = lists:foldl(Fold, [], Vals),
+    lists:keyreplace(port_env, 1, Config, {port_env, NewVals})
+end.
+
 %% Set the minimum Mac target to 10.8 for 10.9 or greater.  This runtime
 %% check is needed since rebar's system_architecture check looks only at
 %% the version of the OS used to build the Erlang runtime, not the version
 %% actually running.
 case os:type() of
-    {unix,darwin} ->
-        Opt = " -mmacosx-version-min=10.8 -stdlib=libc++",
-        [Mjr|_] = string:tokens(os:cmd("/usr/bin/uname -r"), "."),
+    {unix, darwin} ->
+        NewerOsxOpts = " -mmacosx-version-min=10.8 -stdlib=libc++",
+        BrewLibsOpts =
+            case os:find_executable("brew") of
+                false ->
+                    "";
+                _Brew ->
+                    HomebrewPrefix = string:trim(os:cmd("brew --prefix")),
+                    " -L" ++ HomebrewPrefix ++ "/lib"
+            end,
+        [Mjr | _] = string:tokens(os:cmd("/usr/bin/uname -r"), "."),
         Major = list_to_integer(Mjr),
+        Flags = ["CFLAGS", "CXXFLAGS", "LDFLAGS"],
         if
             Major >= 13 ->
-                Flags = ["CFLAGS", "CXXFLAGS", "DRV_CFLAGS", "DRV_LDFLAGS"],
-                {port_env, Vals} = lists:keyfind(port_env, 1, CONFIG),
-                Fold = fun({Flag,Val}, Acc) ->
-                               case lists:member(Flag, Flags) of
-                                   true ->
-                                      [{Flag, Val++Opt}|Acc];
-                                   false ->
-                                       Acc
-                               end
-                       end,
-                NewVals = lists:foldl(Fold, [], Vals),
-                lists:keyreplace(port_env, 1, CONFIG, {port_env, NewVals});
+                AddPortEnv(CONFIG, Flags, NewerOsxOpts ++ BrewLibsOpts);
             true ->
-                CONFIG
+                AddPortEnv(CONFIG, Flags, BrewLibsOpts)
         end;
     _ ->
         CONFIG