Merge pull request #391 from AltGr/gh-act-static-bin

Add github action rules for static binaries generation (Linux, Macos)

.ci-macosx.sh

@@ -11,3 +11,6 @@ eval $(opam env)
 
 opam install -y -j 2 . --deps-only --locked
 make && make opaminstall
+
+# See src/main/linking_flags.sh
+make detect-libs

.github/workflows/static-builds.yml

@@ -0,0 +1,77 @@
+name: Generate static binaries
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - '**'
+jobs:
+  static-bin-linux:
+    name: Builds static Linux binaries
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Build the binaries
+        run: |
+          ./scripts/static-build.sh
+      - name: Test the binaries
+        run: |
+          bin=(./learn-ocaml-client ./learn-ocaml-server ./learn-ocaml)
+          file "${bin[@]}"
+          ldd "${bin[@]}"
+          for b in "${bin[@]}"; do ( set -x; "$b" --version ); done
+      - name: Archive static binaries
+        uses: actions/upload-artifact@v2
+        with:
+          name: static-binaries-linux
+          path: |
+            learn-ocaml
+            learn-ocaml-server
+            learn-ocaml-client
+  static-bin-macos:
+    name: Builds static Macos binaries
+    runs-on: macos-latest
+    env:
+      OPAMYES: 1
+      OPAMDEPEXTYES: 1
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Show OS version
+        run: |
+          sw_vers
+          system_profiler SPSoftwareDataType
+          uname -a
+      # Need unreleased 2.1.0~rc
+      # - name: Retrieve opam
+      #   run: |
+      #     mkdir "$HOME/bin"
+      #     wget https://github.com/ocaml/opam/releases/download/2.1.0-beta2/opam-2.1.0-beta2-x86_64-macos -O $HOME/bin/opam
+      #     chmod a+x $HOME/bin/opam
+      #     echo "$HOME/bin" >> $GITHUB_PATH
+      - name: Install latest opam
+        run: |
+          brew install opam --HEAD
+      - name: Prepare build environment
+        run: |
+          opam init -a --bare
+          opam switch create . ocaml-base-compiler 'dune<2' --deps-only
+      - name: Build the binaries
+        run: |
+          opam exec -- make LINKING_MODE=static
+      - name: Test the binaries
+        run: |
+          bin=(./learn-ocaml-client ./learn-ocaml-server ./learn-ocaml)
+          dir="_build/install/default/bin"
+          file "$dir"/*
+          otool -L "$dir"/*
+          for b in "${bin[@]}"; do ( set -x; "$dir/$b" --version ); done
+      - name: Archive static binaries
+        uses: actions/upload-artifact@v2
+        with:
+          name: static-binaries-macos
+          path: _build/install/default/bin/*

.gitignore

@@ -35,3 +35,5 @@ translations/*.pot
 **/.merlin
 
 tests/corpuses/*
+
+detect-libs.*

Makefile

@@ -90,3 +90,25 @@ travis: # From https://stackoverflow.com/questions/21053657/how-to-run-travis-ci
 	INSTANCE="travisci/ci-garnet:packer-1512502276-986baf0";	\
 	docker run --name $$BUILDID -dit $$INSTANCE /sbin/init &&	\
 	docker exec -it $$BUILDID bash -l
+
+.PHONY: static-binaries
+static-binaries:
+	./scripts/static-build.sh
+
+BINARIES = src/main/learnocaml_client.bc src/main/learnocaml_main.bc src/main/learnocaml_server_main.exe
+
+.PHONY: detect-libs
+detect-libs:
+	$(RM) $(addprefix _build/default/,$(BINARIES))
+	+sort=false; \
+	baseid="detect-libs.$$$$"; echo ...; \
+	$(MAKE) LINKING_MODE=dynamic OCAMLPARAM="_,verbose=1" > $$baseid.log 2>&1; \
+	for bin in $(BINARIES); do \
+	  base=$${bin#src/main/}; base=$${base%.*}; \
+	  grep -e "'$$bin'" $$baseid.log > $$baseid.$$base.log; \
+	  printf "%s: " "$$base"; \
+	  ( sed -e "s/'//g; s/ /\\$$(printf '\n/g')" $$baseid.$$base.log | grep -e "^-l" | \
+	    if [ "$$sort" = true ]; then printf "(sorted) "; sort -u; else cat; fi | xargs echo ); \
+	done; echo; \
+	cat $$baseid.*.log; \
+	$(RM) $$baseid.*log

learn-ocaml-client.opam

@@ -25,7 +25,7 @@ depends: [
   "cohttp-lwt-unix" {>= "1.0.0" & < "2.0.0"}
   "ssl" {= "0.5.5"}
   "digestif" {>= "0.7.1"}
-  "dune" {= "2.0.1"}
+  "dune" {>= "1.11.4" & <= "2.0.1"}
   "ezjsonm"
   "lwt" {>= "4.0.0"}
   "lwt_ssl"

learn-ocaml.opam

@@ -23,7 +23,7 @@ depends: [
   "conf-git"
   "decompress" {= "0.8.1"}
   "digestif" {>= "0.7.1"}
-  "dune" {= "2.0.1"}
+  "dune" {>= "1.11.4"}
   "easy-format" {>= "1.3.0" }
   "ipaddr" {= "2.8.0" }
   "ezjsonm"

scripts/static-build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -ue
+
+LC_ALL=C
+
+cd $(dirname "$0")/..
+
+## Run build in container
+
+set -o pipefail
+git ls-files -z | xargs -0 tar c | \
+  docker run --rm -i \
+    ocamlpro/ocaml:4.05 \
+    sh -uexc \
+      'tar x >&2 &&
+       sudo apk add openssl-libs-static >&2 &&
+       opam switch create . ocaml-system "dune<2" --deps-only >&2 &&
+       opam exec make LINKING_MODE=static >&2 &&
+       tar c -hC _build/install/default/bin .' | \
+  tar vx

src/main/dune

@@ -11,7 +11,8 @@
  (name learnocaml_main)
  (modes byte)
  (ocamlc_flags :standard -custom)
- (flags :standard -linkall)
+ (flags (:standard -linkall
+         (:include linking_main.sexp)))
  (modules Learnocaml_main)
  (libraries cmdliner
             sha
@@ -29,7 +30,8 @@
  (name learnocaml_client)
  (modes byte)
  (ocamlc_flags :standard -custom)
- (flags :standard -linkall)
+ (flags (:standard -linkall
+         (:include linking_client.sexp)))
  (modules Learnocaml_client)
  (libraries cmdliner
  	    sha
@@ -48,4 +50,19 @@
  (name learnocaml_server_main)
  (modules learnocaml_server_main)
  (libraries learnocaml_server_args)
+ (flags       (:standard
+               (:include linking_server.sexp)))
 )
+
+(rule
+  (targets linking_main.sexp)
+  (action  (with-stdout-to %{targets}
+    (run ./linking_flags.sh %{env:LINKING_MODE=dynamic} %{ocaml-config:system} laolao_stubs threads camlrun))))
+(rule
+  (targets linking_client.sexp)
+  (action  (with-stdout-to %{targets}
+    (run ./linking_flags.sh %{env:LINKING_MODE=dynamic} %{ocaml-config:system} threads camlrun))))
+(rule
+  (targets linking_server.sexp)
+  (action  (with-stdout-to %{targets}
+    (run ./linking_flags.sh %{env:LINKING_MODE=dynamic} -- laolao_stubs threadsnat))))

src/main/linking_flags.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+set -ue
+
+# This script is called by dune to generate the linking flags for static builds
+# (on the limited set of supported platforms). It only returns an empty set of
+# flags for the default dynamic linking mode.
+
+LC_ALL=C
+
+help_exit() {
+    echo "Usage: $0 dynamic|static linux|macosx [extra-libs]" >&2
+    exit 2
+}
+
+[ $# -lt 2 ] && help_exit
+
+echo ";; generated by $0"
+
+case "$1" in
+    dynamic) echo "()"; exit 0;;
+    static) ;;
+    *) echo "Invalid linking mode '$1'."; help_exit
+esac
+
+shift
+case "$1" in
+    macosx) shift; EXTRA_LIBS="curses $*";;
+    linux) shift; EXTRA_LIBS="$*";;
+    --) shift; EXTRA_LIBS="$*";;
+    *) echo "Not supported %{ocamlc-config:system} '$1'."; help_exit
+esac
+
+## Static linking configuration ##
+
+# The linked C libraries list may need updating on changes to the dependencies.
+#
+# To get the correct list for manual linking, the simplest way is to set the
+# flags to `-verbose`, while on the normal `autolink` mode, then extract them
+# from the gcc command-line.
+# The Makefile contains a target to automate this: `make detect-libs`.
+
+case $(uname -s) in
+    Linux)
+        case $(. /etc/os-release && echo $ID) in
+            alpine)
+                COMMON_LIBS="camlstr base_stubs ssl_threads_stubs ssl crypto cstruct_stubs lwt_unix_stubs bigarray unix c"
+                # `m` and `pthread` are built-in musl
+                echo '(-noautolink'
+                echo ' -cclib -Wl,-Bstatic'
+                echo ' -cclib -static-libgcc'
+                for l in $EXTRA_LIBS $COMMON_LIBS; do
+                    echo " -cclib -l$l"
+                done
+                echo ' -cclib -static)'
+                ;;
+            *)
+                echo "Error: static linking is only supported in Alpine, to avoids glibc constraints" >&2
+                exit 3
+        esac
+        ;;
+    Darwin)
+        COMMON_LIBS="camlstr base_stubs ssl_threads_stubs /usr/local/opt/openssl/lib/libssl.a /usr/local/opt/openssl/lib/libcrypto.a cstruct_stubs lwt_unix_stubs bigarray unix"
+        # `m` and `pthread` are built-in in libSystem
+        echo '(-noautolink'
+        for l in $EXTRA_LIBS $COMMON_LIBS; do
+            if [ "${l%.a}" != "${l}" ]; then echo " -cclib $l"
+            else echo " -cclib -l$l"
+            fi
+        done
+        echo ')'
+        ;;
+    *)
+        echo "Static linking is not supported for your platform. See $0 to contribute." >&2
+        exit 3
+esac