From fa4f3490777b0560b562e714bcf67c416d978987 Mon Sep 17 00:00:00 2001 From: tdstein Date: Thu, 14 Dec 2023 15:12:50 -0500 Subject: [PATCH 1/5] wip --- install.bash | 10 ++++------ scripts/release.bash | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100755 scripts/release.bash diff --git a/install.bash b/install.bash index 0845a90c7..54b98399f 100755 --- a/install.bash +++ b/install.bash @@ -168,12 +168,10 @@ MKTMP=("mktemp" "-d") TAR=("tar") # Variables -NAME="publisher" -PREFIX="/usr/local/bin" -VERSION="0.0.dev6" -URL="https://cdn.posit.co/publisher/releases/tags/v${VERSION}" -TMPDIR=$(execute "${MKTMP[@]}") - +POSIT_PUBLISHER_NAME="publisher" +POSIT_PUBLISHER_VERSION="0.0.dev6" +POSIT_PUBLISHER_URL="https://cdn.posit.co/publisher/releases/tags/v${VERSION}" +POSIT_PUBLISHER_TMPDIR=$(execute "${MKTMP[@]}") # OS specific settings OS="$(uname)" diff --git a/scripts/release.bash b/scripts/release.bash new file mode 100755 index 000000000..0ff356953 --- /dev/null +++ b/scripts/release.bash @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +if [ "${DEBUG:-false}" = true ]; +then + set -x +fi + +REF=${GITHUB_REF: + +cmd=$1 +if [[ -z "$cmd" ]]; then + echo "usage: $0 " + exit 1 +fi +echo "Command: $cmd" 1>&2 + +version=$(./scripts/get-version.bash) +echo "Version: $version" 1>&2 + +name=$(basename "$cmd") +echo "Name: $name" 1>&2 + +platforms=() +while IFS='' read -r line; do platforms+=("$line"); done < <(./scripts/get-platforms.bash) +for platform in "${platforms[@]}" +do + echo "Releasing: $platform" 1>&2 + os=${platform/\/*} # retain the part before the slash + arch=${platform/*\/} # retain the part after the slash + + archive=$(./scripts/get-archive-path.bash "$name" "$version" "$os" "$arch" ) + echo $archive + + +done From 5fda99ffac4ca9af98f82e47f94a8c7037277e66 Mon Sep 17 00:00:00 2001 From: tdstein Date: Mon, 18 Dec 2023 10:42:18 -0500 Subject: [PATCH 2/5] wip --- .github/workflows/release.yaml | 5 +++++ justfile | 8 ++++++++ scripts/release.bash | 14 ++++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d0cab6b07..6bc6bf907 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -22,6 +22,10 @@ jobs: with: fetch-depth: 0 - uses: extractions/setup-just@v1 + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-east-1 - uses: actions/download-artifact@v3 with: name: archives @@ -40,3 +44,4 @@ jobs: files: | archives/**/* packages/**/* + - run: just release diff --git a/justfile b/justfile index 8a1197bfb..0d89d15e8 100644 --- a/justfile +++ b/justfile @@ -216,6 +216,14 @@ pre-release: ./scripts/is-pre-release.bash +# Releses the application. Releases are written to AWS S3. If invoked with `env CI=true` then releases are created for all architectures supported by the Go toolchain. +release: + #!/usr/bin/env bash + set -eou pipefail + {{ _with_debug }} + + just _with_docker ./scripts/release.bash {{ _cmd }} + # Runs the CLI via `go run`. run *args: #!/usr/bin/env bash diff --git a/scripts/release.bash b/scripts/release.bash index 0ff356953..89e8d0a00 100755 --- a/scripts/release.bash +++ b/scripts/release.bash @@ -5,7 +5,14 @@ then set -x fi -REF=${GITHUB_REF: + +# Obtain the Git ref for use in output path. +# +# Use the GITHUB_REF value if supplied. Otherwise, stub a "fake" ref based on the current branch. +# See https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables +ref="${GITHUB_REF:-"heads/$(git rev-parse --abbrev-ref HEAD)"}" +ref=${ref#"refs/"} +echo "Reference: $ref" 1>&2 cmd=$1 if [[ -z "$cmd" ]]; then @@ -29,7 +36,10 @@ do arch=${platform/*\/} # retain the part after the slash archive=$(./scripts/get-archive-path.bash "$name" "$version" "$os" "$arch" ) - echo $archive + echo "Archive: $archive" 1>&2 + object="s3://posit-publisher/$name/releases/$ref/$(basename $archive)" + echo "Object: $object" 1>&2 + aws s3 cp $archive $object > /dev/null 2>&1 done From a8cd4bab54f43b446784c660bb153a140b062805 Mon Sep 17 00:00:00 2001 From: tdstein Date: Tue, 19 Dec 2023 12:56:30 -0500 Subject: [PATCH 3/5] build: adds tooling for uploading to object storage --- .github/workflows/main.yaml | 6 +++ .github/workflows/pull-request.yaml | 6 +++ .github/workflows/release.yaml | 14 +++--- .github/workflows/upload.yaml | 27 ++++++++++++ CONTRIBUTING.md | 32 ++++++++++++-- build/ci/Dockerfile | 10 ++++- install.bash | 10 +++-- justfile | 23 +++++++--- scripts/get-vscode-extension-path.bash | 18 ++++++++ scripts/release.bash | 45 -------------------- scripts/upload.bash | 59 ++++++++++++++++++++++++++ 11 files changed, 183 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/upload.yaml create mode 100755 scripts/get-vscode-extension-path.bash delete mode 100755 scripts/release.bash create mode 100755 scripts/upload.bash diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2c6281882..782fc42f5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -22,6 +22,12 @@ jobs: archive: needs: build uses: ./.github/workflows/archive.yaml + upload: + needs: + - archive + - package + uses: ./.github/workflows/upload.yaml + secrets: inherit # Integration Tests bats: diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index c47edfd95..cb2cad8aa 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -23,6 +23,12 @@ jobs: archive: needs: build uses: ./.github/workflows/archive.yaml + upload: + needs: + - archive + - package + uses: ./.github/workflows/upload.yaml + secrets: inherit # Integration Tests bats: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6bc6bf907..d6e553eb5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,6 +12,12 @@ jobs: needs: - build uses: ./.github/workflows/archive.yaml + upload: + needs: + - archive + - package + uses: ./.github/workflows/upload.yaml + secrets: inherit release: runs-on: ubuntu-latest needs: @@ -22,10 +28,6 @@ jobs: with: fetch-depth: 0 - uses: extractions/setup-just@v1 - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: us-east-1 - uses: actions/download-artifact@v3 with: name: archives @@ -36,12 +38,10 @@ jobs: path: packages - id: get-prerelease run: echo "prerelease=$(just pre-release)" >> "$GITHUB_OUTPUT" - - name: Release - uses: softprops/action-gh-release@v1 + - uses: softprops/action-gh-release@v1 with: draft: false prerelease: ${{ steps.get-prerelease.outputs.prerelease == 'true' }} files: | archives/**/* packages/**/* - - run: just release diff --git a/.github/workflows/upload.yaml b/.github/workflows/upload.yaml new file mode 100644 index 000000000..0bbf26711 --- /dev/null +++ b/.github/workflows/upload.yaml @@ -0,0 +1,27 @@ +name: Upload +on: + workflow_call: +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + upload: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup + - uses: actions/download-artifact@v3 + with: + name: archives + path: archives + - uses: actions/download-artifact@v3 + with: + name: packages + path: packages + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + - run: just upload diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec9dcda2c..161759ea8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,30 @@ ## Quick Start +- [Contributing](#contributing) + - [Quick Start](#quick-start) + - [Getting Started](#getting-started) + - [Prerequisites](#prerequisites) + - [Option 1 - Native](#option-1---native) + - [Option 2 - Docker](#option-2---docker) + - [Installing](#installing) + - [Execution](#execution) + - [Testing](#testing) + - [Unit Tests](#unit-tests) + - [Coverage Reporting](#coverage-reporting) + - [Integration Tests](#integration-tests) + - [UI Tests](#ui-tests) + - [Development](#development) + - [Build Tools](#build-tools) + - [Environment Variables](#environment-variables) + - [Behavior in GitHub Actions](#behavior-in-github-actions) + - [Extension Development](#extension-development) + - [Release](#release) + - [Instructions](#instructions) + - [Pre-Releases](#pre-releases) + - [Release Lifecycle](#release-lifecycle) + + The get this project up and running on your local machine, execute the following Just commands: ```console @@ -106,7 +130,7 @@ When executing commands the following variables are accepted to change behavior. | MODE | dev | enum | When set to `dev`, development is enabled. All other values disable development mode. | -#### Continuous Integration in GitHub Actions +#### Behavior in GitHub Actions When running in GitHub Actions, the env variable `CI` is set to `true` by GitHub. When `CI=true`, the defaults for the following values are adjusted. @@ -125,6 +149,8 @@ Execute `eval "$(just configure)"` to configure the executable on your current ` ## Release +### Instructions + To start a release create a semver compatible tag. _For this example, we will use the tag `v0.0.dev0`. This tag already exists, so you will not be able run the following commands verbatim._ @@ -158,9 +184,7 @@ For additional definitions see https://en.wikipedia.org/wiki/Software_release_li Currently, the following suffix lineage is in use: -## Release Lifecycle - -### Key +### Release Lifecycle - `X`: The major version. - `Y`: The minor version. diff --git a/build/ci/Dockerfile b/build/ci/Dockerfile index 5e541f6f6..8f81305a8 100644 --- a/build/ci/Dockerfile +++ b/build/ci/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM ubuntu:jammy +FROM ubuntu:jammy ARG NODEVERSION=18 RUN \ @@ -55,3 +55,11 @@ RUN\ && go env -w GOPATH=/usr/local/go\ # staticcheck is required to for linting Go && go install honnef.co/go/tools/cmd/staticcheck@latest + +# AWS is required by this project. The build tooling uses it! +ARG AWSARCHIVE=awscli-exe-linux-x86_64.zip +RUN curl -L -O "https://awscli.amazonaws.com/${AWSARCHIVE}" \ + && unzip -q "${AWSARCHIVE}" \ + && rm "${AWSARCHIVE}"\ + && ./aws/install\ + && rm -r ./aws diff --git a/install.bash b/install.bash index 54b98399f..222becfcd 100755 --- a/install.bash +++ b/install.bash @@ -168,10 +168,12 @@ MKTMP=("mktemp" "-d") TAR=("tar") # Variables -POSIT_PUBLISHER_NAME="publisher" -POSIT_PUBLISHER_VERSION="0.0.dev6" -POSIT_PUBLISHER_URL="https://cdn.posit.co/publisher/releases/tags/v${VERSION}" -POSIT_PUBLISHER_TMPDIR=$(execute "${MKTMP[@]}") +NAME="publisher" +PREFIX="/usr/local/bin" +VERSION="1.0.dev0" +URL="https://cdn.posit.co/publisher/releases/tags/v${VERSION}" +TMPDIR=$(execute "${MKTMP[@]}") + # OS specific settings OS="$(uname)" diff --git a/justfile b/justfile index 0d89d15e8..4bf02747c 100644 --- a/justfile +++ b/justfile @@ -50,6 +50,8 @@ default: just clean just web just build + just package + just archive # Executes command against every justfile where avaiable. WARNING your mileage may very. all +args='default': @@ -208,21 +210,20 @@ name: basename {{ _cmd }} -# Prints the pre-release status based on the version (see `just version`). -pre-release: +package: #!/usr/bin/env bash set -eou pipefail {{ _with_debug }} - ./scripts/is-pre-release.bash + just vscode package -# Releses the application. Releases are written to AWS S3. If invoked with `env CI=true` then releases are created for all architectures supported by the Go toolchain. -release: +# Prints the pre-release status based on the version (see `just version`). +pre-release: #!/usr/bin/env bash set -eou pipefail {{ _with_debug }} - just _with_docker ./scripts/release.bash {{ _cmd }} + ./scripts/is-pre-release.bash # Runs the CLI via `go run`. run *args: @@ -263,6 +264,14 @@ test *args=("./..."): stub just _with_docker go test {{ args }} -covermode set -coverprofile=cover.out +# Uploads distributions to object storage. If invoked with `env CI=true` then all architectures supported by the Go toolchain are uploaded. +upload: + #!/usr/bin/env bash + set -eou pipefail + {{ _with_debug }} + + just _with_docker ./scripts/upload.bash {{ _cmd }} + # Executes commands in ./web/Justfile. Equivalent to `just web/dist`, but inside of Docker (i.e., just _with_docker web/dist). web *args: #!/usr/bin/env bash @@ -309,6 +318,8 @@ _with_docker *args: -e GOCACHE=/work/.cache/go/cache\ -e GOMODCACHE=/work/.cache/go/mod\ -e MODE={{ _mode }}\ + --env-file <(env | grep AWS_)\ + --env-file <(env | grep GITHUB_)\ --platform {{ _docker_platform }}\ -v "$(pwd)":/work\ -w /work\ diff --git a/scripts/get-vscode-extension-path.bash b/scripts/get-vscode-extension-path.bash new file mode 100755 index 000000000..fc904e73c --- /dev/null +++ b/scripts/get-vscode-extension-path.bash @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +if [ "${DEBUG:-false}" = true ]; +then + set -x +fi + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " + exit 1 +fi + +cmd=$1 +version=$2 + +name=$(basename "$cmd") + +printf "$(pwd)/packages/%s" "$name-$version.vsix" diff --git a/scripts/release.bash b/scripts/release.bash deleted file mode 100755 index 89e8d0a00..000000000 --- a/scripts/release.bash +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -if [ "${DEBUG:-false}" = true ]; -then - set -x -fi - - -# Obtain the Git ref for use in output path. -# -# Use the GITHUB_REF value if supplied. Otherwise, stub a "fake" ref based on the current branch. -# See https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables -ref="${GITHUB_REF:-"heads/$(git rev-parse --abbrev-ref HEAD)"}" -ref=${ref#"refs/"} -echo "Reference: $ref" 1>&2 - -cmd=$1 -if [[ -z "$cmd" ]]; then - echo "usage: $0 " - exit 1 -fi -echo "Command: $cmd" 1>&2 - -version=$(./scripts/get-version.bash) -echo "Version: $version" 1>&2 - -name=$(basename "$cmd") -echo "Name: $name" 1>&2 - -platforms=() -while IFS='' read -r line; do platforms+=("$line"); done < <(./scripts/get-platforms.bash) -for platform in "${platforms[@]}" -do - echo "Releasing: $platform" 1>&2 - os=${platform/\/*} # retain the part before the slash - arch=${platform/*\/} # retain the part after the slash - - archive=$(./scripts/get-archive-path.bash "$name" "$version" "$os" "$arch" ) - echo "Archive: $archive" 1>&2 - - object="s3://posit-publisher/$name/releases/$ref/$(basename $archive)" - echo "Object: $object" 1>&2 - - aws s3 cp $archive $object > /dev/null 2>&1 -done diff --git a/scripts/upload.bash b/scripts/upload.bash new file mode 100755 index 000000000..cddae951d --- /dev/null +++ b/scripts/upload.bash @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail +if [ "${DEBUG:-false}" = true ]; +then + set -x +fi + +cmd=$1 +if [[ -z "$cmd" ]]; then + echo "usage: $0 " + exit 1 +fi +echo "Command: $cmd" 1>&2 + +version=$(./scripts/get-version.bash) +echo "Version: $version" 1>&2 + +name=$(basename "$cmd") +echo "Name: $name" 1>&2 + +ref="${GITHUB_REF}" +echo "Git Reference: $ref" 1>&2 +echo "Git Commit Hash: $(git show-ref --heads --hash "$ref")" +ref=${ref#"refs/"} + +object_path="s3://posit-publisher/$name/releases/$ref" +echo "Object Path: $object_path" 1>&2 + +platforms=() +while IFS='' read -r line; do platforms+=("$line"); done < <(./scripts/get-platforms.bash) +for platform in "${platforms[@]}" +do + echo + echo "Release: $platform" 1>&2 + os=${platform/\/*} # retain the part before the slash + arch=${platform/*\/} # retain the part after the slash + archive=$(./scripts/get-archive-path.bash "$name" "$version" "$os" "$arch" ) + echo "Archive: $archive" 1>&2 + if ! [ -f "$archive" ]; + then + echo "Not Found. Skipping..." 1>&2 + else + object="$object_path/$(basename "$archive")" + echo "Object: $object" 1>&2 + aws s3 cp "$archive" "$object" > /dev/null 2>&1 + fi +done + +extension=$(./scripts/get-vscode-extension-path.bash "$name" "$version") +echo +echo "VSCode Extension: $extension" +if ! [ -f "$extension" ]; +then + echo "Not Found. Skipping..." 1>&2 +else + object="$object_path/$(basename "$extension")" + echo "Object: $object" 1>&2 + aws s3 cp "$extension" "$object" > /dev/null 2>&1 +fi From c331a769c851a235e83f451d380fe42772ab4d6f Mon Sep 17 00:00:00 2001 From: tdstein Date: Mon, 29 Jan 2024 12:39:29 -0500 Subject: [PATCH 4/5] fix: minor pull request feedback --- scripts/get-vscode-extension-path.bash | 2 +- scripts/upload.bash | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/get-vscode-extension-path.bash b/scripts/get-vscode-extension-path.bash index fc904e73c..e218460ed 100755 --- a/scripts/get-vscode-extension-path.bash +++ b/scripts/get-vscode-extension-path.bash @@ -15,4 +15,4 @@ version=$2 name=$(basename "$cmd") -printf "$(pwd)/packages/%s" "$name-$version.vsix" +echo "$(pwd)/packages/$name-$version.vsix" diff --git a/scripts/upload.bash b/scripts/upload.bash index cddae951d..161821c28 100755 --- a/scripts/upload.bash +++ b/scripts/upload.bash @@ -20,7 +20,6 @@ echo "Name: $name" 1>&2 ref="${GITHUB_REF}" echo "Git Reference: $ref" 1>&2 -echo "Git Commit Hash: $(git show-ref --heads --hash "$ref")" ref=${ref#"refs/"} object_path="s3://posit-publisher/$name/releases/$ref" From 003dd28e4f3d6bc69c7201df0102a03f98a89aec Mon Sep 17 00:00:00 2001 From: tdstein Date: Thu, 1 Feb 2024 15:47:35 -0500 Subject: [PATCH 5/5] build: adds upload support for multiplatform builds --- scripts/get-vscode-extension-path.bash | 8 +++++--- scripts/upload.bash | 25 ++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/scripts/get-vscode-extension-path.bash b/scripts/get-vscode-extension-path.bash index e218460ed..960224d8e 100755 --- a/scripts/get-vscode-extension-path.bash +++ b/scripts/get-vscode-extension-path.bash @@ -5,14 +5,16 @@ then set -x fi -if [ "$#" -ne 2 ]; then - echo "usage: $0 " +if [ "$#" -ne 4 ]; then + echo "usage: $0 " exit 1 fi cmd=$1 version=$2 +os=$3 +arch=$4 name=$(basename "$cmd") -echo "$(pwd)/packages/$name-$version.vsix" +echo "$(pwd)/packages/$name-$version-$os-$arch.vsix" diff --git a/scripts/upload.bash b/scripts/upload.bash index 161821c28..72b752644 100755 --- a/scripts/upload.bash +++ b/scripts/upload.bash @@ -18,7 +18,7 @@ echo "Version: $version" 1>&2 name=$(basename "$cmd") echo "Name: $name" 1>&2 -ref="${GITHUB_REF}" +ref="${GITHUB_REF:-$(git show-ref --heads "$(git rev-parse --abbrev-ref HEAD)" | awk '{print $2}')}" echo "Git Reference: $ref" 1>&2 ref=${ref#"refs/"} @@ -43,16 +43,15 @@ do echo "Object: $object" 1>&2 aws s3 cp "$archive" "$object" > /dev/null 2>&1 fi -done -extension=$(./scripts/get-vscode-extension-path.bash "$name" "$version") -echo -echo "VSCode Extension: $extension" -if ! [ -f "$extension" ]; -then - echo "Not Found. Skipping..." 1>&2 -else - object="$object_path/$(basename "$extension")" - echo "Object: $object" 1>&2 - aws s3 cp "$extension" "$object" > /dev/null 2>&1 -fi + extension=$(./scripts/get-vscode-extension-path.bash "$name" "$version" "$os" "$arch") + echo "VSCode Extension: $extension" + if ! [ -f "$extension" ]; + then + echo "Not Found. Skipping..." 1>&2 + else + object="$object_path/$(basename "$extension")" + echo "Object: $object" 1>&2 + aws s3 cp "$extension" "$object" > /dev/null 2>&1 + fi +done