feat(ci/workflows): add build_artifacts (#2)

Signed-off-by: Hanchin Hsieh <[email protected]>

.github/scripts/matrix.yaml

@@ -0,0 +1,75 @@
+os: ["ubuntu-latest"]
+build:
+  - target: "x86_64-unknown-linux-gnu"
+    cc: "gcc"
+    goos: "linux"
+    goarch: "amd64"
+  - target: "aarch64-unknown-linux-gnu"
+    cc: "aarch64-linux-gnu-gcc"
+    goos: "linux"
+    goarch: "arm64"
+service:
+  - "aliyun-drive"
+  # - "alluxio"
+  # - "atomicserver"
+  # - "azblob"
+  # - "azdls"
+  # - "azfile"
+  # - "b2"
+  # - "cacache"
+  # - "chainsafe"
+  # - "cloudflare-kv"
+  # - "compfs"
+  # - "cos"
+  # - "d1"
+  # - "dashmap"
+  # - "dbfs"
+  # - "dropbox"
+  # - "etcd"
+  # - "foundationdb"
+  - "fs"
+  # - "ftp"
+  # - "gcs"
+  - "gdrive"
+  # - "ghac"
+  # - "github"
+  # - "gridfs"
+  # - "hdfs-native"
+  # - "http"
+  # - "huggingface"
+  # - "icloud"
+  # - "ipmfs"
+  # - "koofr"
+  # - "libsql"
+  # - "memcached"
+  - "memory"
+  # - "mini-moka"
+  # - "moka"
+  # - "mongodb"
+  # - "mysql"
+  # - "obs"
+  # - "onedrive"
+  # - "oss"
+  # - "pcloud"
+  # - "persy"
+  # - "postgresql"
+  # - "redb"
+  # - "redis"
+  # - "redis-native-tls"
+  # - "rocksdb"
+  - "s3"
+  # - "seafile"
+  # - "sftp"
+  # - "sled"
+  # - "sqlite"
+  # - "supabase"
+  # - "surrealdb"
+  # - "swift"
+  # - "tikv"
+  # - "upyun"
+  # - "vercel-artifacts"
+  # - "vercel-blob"
+  # - "webdav"
+  # - "webhdfs"
+  # - "yandex-disk"
+

.github/scripts/setup_features.py

@@ -0,0 +1,19 @@
+import toml
+import os
+
+cargoPath = 'bindings/c/Cargo.toml'
+
+with open(cargoPath, 'r') as f:
+    cargo_toml = toml.load(f)
+
+enabled_features = os.environ.get('OPENDAL_FEATURES', '').split(',')
+
+opendal_features = []
+for feature in enabled_features:
+    if feature.strip():
+        opendal_features.append(feature.strip())
+
+cargo_toml['dependencies']['opendal']['features'] = opendal_features
+
+with open(cargoPath, 'w') as f:
+    toml.dump(cargo_toml, f)

.github/workflows/build_artifacts.yaml

@@ -0,0 +1,113 @@
+name: build_artifacts
+on:
+  workflow_dispatch:
+    inputs:
+      opendal_core_version:
+        description: "Version of Apache OpenDAL Core"
+        required: true
+        type: string
+      opendal_go_version:
+        description: "Version of Apache OpenDAL Go Binding"
+        required: false
+        type: string
+jobs:
+  matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: set-matrix
+        name: Setup Matrix
+        run: |
+          MATRIX=$(yq -o=json -I=0 "." .github/scripts/matrix.yaml | sed 's/ //g')
+          echo "Matrix:"
+          echo "$MATRIX" | jq .
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+  build:
+    needs: [ matrix ]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: "apache/opendal"
+          ref: ${{ inputs.opendal_core_version }}
+      - uses: actions/checkout@v4
+        with:
+          path: "tools"
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+      - name: Setup Target
+        env:
+          TARGET: ${{ matrix.build.target }}
+        run: rustup target add $TARGET
+      - name: Setup AArch64 Tool
+        working-directory: bindings/c
+        if: ${{ matrix.build.target == 'aarch64-unknown-linux-gnu' }}
+        run: |
+          sudo apt update
+          sudo apt install gcc-aarch64-linux-gnu -y
+          mkdir .cargo
+          cat << EOF > .cargo/config.toml
+          [target.aarch64-unknown-linux-gnu]
+          linker = "aarch64-linux-gnu-gcc"
+          EOF
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Setup Service
+        env:
+          OPENDAL_FEATURES: "layers-blocking,services-${{ matrix.service }}"
+        run: |
+          python -m pip install toml
+          python tools/.github/scripts/setup_features.py
+      - name: Build ${{ matrix.service }} ${{ matrix.build.target }}
+        working-directory: bindings/c
+        env:
+          SERVICE: ${{ matrix.service }}
+          TARGET: ${{ matrix.build.target }}
+          CC: ${{ matrix.build.cc }}
+        run: |
+          cargo build --target $TARGET  --release
+          sudo apt install zstd
+          zstd -22 ./target/$TARGET/release/libopendal_c.so -o ./libopendal_c.$TARGET.so.zst
+      - uses: actions/upload-artifact@v4
+        with:
+          name: "libopendal_c_${{ inputs.opendal_core_version }}_${{ matrix.service }}_${{ matrix.build.target }}"
+          if-no-files-found: "error"
+          path: "bindings/c/libopendal_c.${{ matrix.build.target }}.so.zst"
+          overwrite: "true"
+  generate:
+    needs: [ matrix, build ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+      - uses: actions/download-artifact@v4
+      - name: Generate Template
+        env:
+          MATRIX: ${{ needs.matrix.outputs.matrix }}
+          VERSION: ${{ inputs.opendal_core_version }}
+        working-directory: generate
+        run: |
+          go run generate.go
+      - name: Auto Commit
+        env:
+          VERSION: ${{ inputs.opendal_core_version }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config --global user.name "Github Actions"
+          git config --global user.email "[email protected]"
+          git add -A
+          git commit -m "Auto commit by GitHub Actions $VERSION"
+          git push -f --set-upstream origin main
+      - name: Auto Tag
+        if: ${{ inputs.opendal_go_version != '' }}
+        env:
+          TAG: ${{ inputs.opendal_go_version }}
+        run: |
+          git tag $TAG
+          git push -f origin $TAG

generate/generate.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"text/template"
+)
+
+type Build struct {
+	Target string `json:"target"`
+	CC     string `json:"cc"`
+	GOOS   string `json:"goos"`
+	GOARCH string `json:"goarch"`
+}
+
+type Matrix struct {
+	Builds   []Build  `json:"build"`
+	Services []string `json:"service"`
+}
+
+var (
+	matrix  Matrix
+	version string
+
+	tpls = template.Must(template.ParseGlob("templates/*.tpl"))
+)
+
+func init() {
+	json.Unmarshal([]byte(os.Getenv("MATRIX")), &matrix)
+	version = os.Getenv("VERSION")
+}
+
+func genGoFile(build Build, service string) error {
+	pkg := strings.ReplaceAll(service, "-", "_")
+	pkgPath := fmt.Sprintf("../%s", pkg)
+
+	_, err := os.Stat(pkgPath)
+	if os.IsNotExist(err) {
+		err := os.Mkdir(pkgPath, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = os.Rename(
+		fmt.Sprintf("../libopendal_c_%s_%s_%s/libopendal_c.%s.so.zst", version, service, build.Target, build.Target),
+		fmt.Sprintf("../%s/libopendal_c.%s.%s.so.zst", pkg, build.GOOS, build.GOARCH))
+	if err != nil {
+		return err
+	}
+
+	for _, t := range tpls.Templates() {
+		fileTpl := template.Must(template.New("file").Parse(t.Name()))
+		var buf bytes.Buffer
+		err := fileTpl.Execute(&buf, map[string]string{
+			"os":   build.GOOS,
+			"arch": build.GOARCH,
+		})
+		if err != nil {
+			return fmt.Errorf("parse filename: %s:%s", t.Name(), err)
+		}
+		targetFile := fmt.Sprintf("../%s/%s", pkg, strings.Trim(buf.String(), ".tpl"))
+		os.Remove(targetFile)
+
+		file, err := os.OpenFile(targetFile, os.O_CREATE|os.O_WRONLY, os.ModePerm)
+		if err != nil {
+			return fmt.Errorf("open file: %s: %s", t.Name(), err)
+		}
+		defer file.Close()
+
+		if err := t.Execute(file, map[string]string{
+			"pkg":  pkg,
+			"os":   build.GOOS,
+			"arch": build.GOARCH,
+		}); err != nil {
+			return fmt.Errorf("execute template: %s: %s", t.Name(), err)
+		}
+	}
+	cmd := exec.Command("go", "mod", "tidy")
+	cmd.Dir = pkgPath
+	output, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%s\n", output)
+	return nil
+}
+
+func main() {
+	for _, service := range matrix.Services {
+		for _, build := range matrix.Builds {
+			err := genGoFile(build, service)
+			if err != nil {
+				panic(fmt.Errorf("failed to generate go file: %s", err))
+			}
+		}
+	}
+}

generate/go.mod

@@ -0,0 +1,3 @@
+module generate
+
+go 1.22.5

generate/templates/embed_{{.os}}_{{.arch}}.go.tpl

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// generated by github.com/apache/opendal-go-services
+
+package {{.pkg}}
+
+import _ "embed"
+
+//go:embed libopendal_c.{{.os}}.{{.arch}}.so.zst
+var libopendalZst []byte

generate/templates/go.mod.tpl

@@ -0,0 +1,14 @@
+module github.com/apache/opendal-go-services/{{.pkg}}
+
+go 1.22.4
+
+require (
+	github.com/klauspost/compress v1.17.9
+	github.com/stretchr/testify v1.9.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)