diff --git a/.gitignore b/.gitignore index 738e9ba..a0ccaec 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ venv.bak/ .DS_Store .vscode + +*.private.yml +*.private.yaml diff --git a/jobs/ci-run/ci-run-master.yml b/jobs/ci-run/ci-run-master.yml index 7c0631e..42ad984 100644 --- a/jobs/ci-run/ci-run-master.yml +++ b/jobs/ci-run/ci-run-master.yml @@ -1,30 +1,9 @@ - job: - name: "ci-build-juju" + name: "ci-trigger-github-com-juju-juju" disabled: false concurrent: false - description: | - Build Juju for subsequent CI run jobs to use. -
- The rough outline of a build stage is: - project-type: "multijob" node: noop-parent-jobs - # Any changes to the refspec for branches watched must be updated in the - # 'generate-testing-streams' job otherwise they will never persist in - # streams. - # properties: - github: url: https://github.com/juju/juju/ @@ -62,7 +41,39 @@ false + publishers: + - trigger-parameterized-builds: + - project: "ci-build-juju" + condition: SUCCESS + current-parameters: false + predefined-parameters: |- + GITHUB_REPO=juju/juju + GITHUB_BRANCH_HEAD_SHA=${{GITHUB_BRANCH_HEAD_SHA}} + +- job: + name: "ci-build-juju" + disabled: false + concurrent: false + description: | + Build Juju for subsequent CI run jobs to use. +
+ The rough outline of a build stage is: + + project-type: "multijob" + node: noop-parent-jobs wrappers: - ansicolor - workspace-cleanup @@ -71,28 +82,36 @@ timeout: 120 fail: true type: absolute - + - credentials-binding: + - text: + credential-id: github-token-private + variable: GITHUB_TOKEN parameters: - string: + name: GITHUB_REPO + default: juju/juju + description: "Specifc github repo to pull from" + - string: + name: GITHUB_BRANCH_HEAD_SHA default: "" description: "Specific git SHA to build (used to overwrite triggered runs)." - name: GITHUB_BRANCH_HEAD_SHA - builders: - resolve-git-sha - detect-commit-go-version + - inject: + properties-content: |- + GITHUB_TOKEN= - multijob: name: "Packaging" condition: SUCCESSFUL projects: - name: "package-juju-source" - current-parameters: true + current-parameters: false predefined-parameters: |- + GITHUB_REPO=${{GITHUB_REPO}} GITHUB_BRANCH_HEAD_SHA=${{GITHUB_BRANCH_HEAD_SHA}} - SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}} JUJU_BUILD_NUMBER=${{BUILD_NUMBER}} GOVERSION=${{GOVERSION}} - - get-s3-source-payload - inject: properties-content: |- @@ -100,7 +119,7 @@ series=focal - get-build-details - description-setter: - description: "${{JUJU_VERSION}}:${{SHORT_GIT_COMMIT}} (go ${{GOVERSION}})" + description: "${{GITHUB_REPO}} ${{JUJU_VERSION}}:${{SHORT_GIT_COMMIT}} (go ${{GOVERSION}})" - multijob: name: "Building Juju Binaries" condition: SUCCESSFUL @@ -172,7 +191,6 @@ BUILD_LABEL=k8s-linux-amd64 SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}} GOVERSION=${{GOVERSION}} - - multijob: name: "Building Juju OCI Images" condition: SUCCESSFUL @@ -182,7 +200,6 @@ predefined-parameters: |- SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}} GOVERSION=${{GOVERSION}} - - multijob: name: "Simple Streams" condition: SUCCESSFUL @@ -192,7 +209,6 @@ current-parameters: true predefined-parameters: |- SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}} - - multijob: name: "Simple Streams Publish" condition: SUCCESSFUL @@ -206,7 +222,6 @@ current-parameters: true predefined-parameters: |- SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}} - publishers: - trigger-parameterized-builds: - project: "ci-gating-tests" @@ -235,25 +250,24 @@ timeout: 1200 fail: true type: absolute - parameters: - validating-string: - description: The git short hash for the commit you wish to test name: SHORT_GIT_COMMIT + description: The git short hash for the commit you wish to test regex: ^\S{{7}}$ msg: Enter a valid 7 char git sha - string: + name: series default: "" description: "Series to use with charms in the functional tests" - name: series - string: + name: BOOTSTRAP_SERIES default: '' description: 'Ubuntu series to use when bootstrapping Juju' - name: BOOTSTRAP_SERIES - string: + name: GOVERSION default: '' description: 'Go version used for build.' - name: GOVERSION builders: - get-build-details - set-test-description @@ -296,24 +310,37 @@ fail: true timeout: 240 # Might have to queue for nodes, give some flex type: absolute + - credentials-binding: + - ssh-user-private-key: + credential-id: github-pull-ssh-key + key-file-variable: GITHUB_SSH_KEY parameters: + - string: + name: GITHUB_REPO + default: "" + description: "Specifc github repo to pull from." - string: name: GITHUB_BRANCH_HEAD_SHA default: "" description: "Specific git SHA to package." - string: - default: '' - description: 'Go version used for build.' name: GOVERSION + default: "" + description: 'Go version used for build.' + - string: + name: JUJU_BUILD_NUMBER + default: "" + description: "Juju build number." builders: - install-go - inject: properties-content: |- JUJU_SOURCE_CHECKOUT=${{WORKSPACE}}/tmp_initial_clone - shell: |- + export GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -i $GITHUB_SSH_KEY -F /dev/null" git init "$JUJU_SOURCE_CHECKOUT" cd "$JUJU_SOURCE_CHECKOUT" - git remote add origin "https://github.com/juju/juju.git" + git remote add origin "git@github.com:$GITHUB_REPO.git" git config --local gc.auto 0 git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin "+$GITHUB_BRANCH_HEAD_SHA" git checkout --progress --force $GITHUB_BRANCH_HEAD_SHA diff --git a/jobs/ci-run/scripts/goversion.sh b/jobs/ci-run/scripts/goversion.sh index f9600ad..1a968bd 100644 --- a/jobs/ci-run/scripts/goversion.sh +++ b/jobs/ci-run/scripts/goversion.sh @@ -1,7 +1,13 @@ #!/bin/bash set -ex -gomod=$(curl -s "https://raw.githubusercontent.com/juju/juju/$GIT_COMMIT/go.mod") +GITHUB_REPO=${GITHUB_REPO:-juju/juju} + +if [ -z "${GITHUB_TOKEN}" ]; then +gomod=$(curl -s "https://raw.githubusercontent.com/$GITHUB_REPO/$GIT_COMMIT/go.mod") +else +gomod=$(curl -s "https://api.github.com/repos/$GITHUB_REPO/contents/go.mod?ref=$GIT_COMMIT" --header "Authorization: Bearer $GITHUB_TOKEN" | jq ".content" -r | base64 -d) +fi goversion=$(echo "$gomod" | grep "go 1." | sed 's/^go 1\.\(.*\)$/1.\1/') if [[ "$goversion" < "1.14" ]]; then diff --git a/jobs/ci-run/scripts/package-juju-source.sh b/jobs/ci-run/scripts/package-juju-source.sh index a25a510..f20a42c 100644 --- a/jobs/ci-run/scripts/package-juju-source.sh +++ b/jobs/ci-run/scripts/package-juju-source.sh @@ -1,11 +1,6 @@ #!/bin/bash set -xe -if [ -z "$GITHUB_BRANCH_NAME" ] && [ -z "$GITHUB_BRANCH_HEAD_SHA" ]; then - echo "Must specify one of GITHUB_BRANCH_NAME or GITHUB_BRANCH_HEAD_SHA" - exit 1 -fi - if [ -z "$JUJU_SOURCE_CHECKOUT" ]; then echo "Must specify JUJU_SOURCE_CHECKOUT" exit 1 @@ -26,15 +21,6 @@ full_path=${GOPATH}/src/github.com/juju/juju export PATH=/snap/bin:$PATH:$GOPATH/bin -# If run with an overriding SHORT_GIT_COMMIT we need to -# determine the branch name ourselves. -if [ -z ${GITHUB_BRANCH_NAME} ]; then - GITHUB_BRANCH_NAME=$(git -C ${JUJU_SOURCE_CHECKOUT} branch --all \ - --contains ${GITHUB_BRANCH_HEAD_SHA} \ - --column \ - | awk '{print$NF}' \ - | awk -F/ '{print$NF}') -fi rm -rf ${full_path} git clone --depth 1 file://${JUJU_SOURCE_CHECKOUT} ${full_path} rm -fr ${JUJU_SOURCE_CHECKOUT} diff --git a/jobs/ci-run/scripts/resolve-git-sha.sh b/jobs/ci-run/scripts/resolve-git-sha.sh index c4e127c..16df667 100644 --- a/jobs/ci-run/scripts/resolve-git-sha.sh +++ b/jobs/ci-run/scripts/resolve-git-sha.sh @@ -1,6 +1,7 @@ #!/bin/bash set -xe +echo "GITHUB_REPO=$GITHUB_REPO" echo "GITHUB_BRANCH_NAME=$GITHUB_BRANCH_NAME" echo "GITHUB_BRANCH_HEAD_SHA=$GITHUB_BRANCH_HEAD_SHA" echo "GIT_COMMIT=$GIT_COMMIT" @@ -14,21 +15,21 @@ function cleanup { trap cleanup EXIT -function query_github_simple { - curl -s "https://api.github.com/repos/juju/juju/commits/$1" | jq -r ".sha // empty" +function query_github_simple { \ + curl -s "https://api.github.com/repos/$GITHUB_REPO/commits/$1" --header "Authorization: Bearer $GITHUB_TOKEN" --header "X-GitHub-Api-Version: 2022-11-28" | jq -r ".sha // empty" } function query_github_treeish { - curl -s "https://api.github.com/repos/juju/juju/git/trees/$1" | jq -r ".sha // empty" + curl -s "https://api.github.com/repos/$GITHUB_REPO/git/trees/$1" --header "Authorization: Bearer $GITHUB_TOKEN" --header "X-GitHub-Api-Version: 2022-11-28" | jq -r ".sha // empty" } function search_github_hash { - curl -s -H "Accept: application/vnd.github.cloak-preview+json" "https://api.github.com/search/commits?q=repo:juju/juju+hash:$1" | jq -r ".items[0].sha // empty" + curl -s -H "Accept: application/vnd.github.cloak-preview+json" "https://api.github.com/search/commits?q=repo:$GITHUB_REPO+hash:$1" --header "Authorization: Bearer $GITHUB_TOKEN" --header "X-GitHub-Api-Version: 2022-11-28" | jq -r ".items[0].sha // empty" } function clone_search { if [ ! -d "$TMP_CLONE" ]; then - git clone -q --no-checkout "https://github.com/juju/juju.git" "$TMP_CLONE" + git clone -q --no-checkout "https://github.com/$GITHUB_REPO.git" "$TMP_CLONE" fi git -C "$TMP_CLONE" rev-parse "$1" } @@ -95,5 +96,6 @@ echo "SHORT_GIT_COMMIT=${EXACT_COMMIT:0:7}" > $PROPS_PATH echo "GIT_COMMIT=${EXACT_COMMIT}" >> $PROPS_PATH echo "GITHUB_BRANCH_HEAD_SHA=${EXACT_COMMIT}" >> $PROPS_PATH echo "GITHUB_BRANCH_NAME=${GITHUB_BRANCH_NAME}" >> $PROPS_PATH +echo "GITHUB_REPO=${GITHUB_REPO}" >> $PROPS_PATH cat $PROPS_PATH \ No newline at end of file diff --git a/tests/suites/static_analysis/deadcode/main.go b/tests/suites/static_analysis/deadcode/main.go index 80d512a..3f7b095 100644 --- a/tests/suites/static_analysis/deadcode/main.go +++ b/tests/suites/static_analysis/deadcode/main.go @@ -190,6 +190,10 @@ func extractMultiJobNames(multijob map[interface{}]interface{}) []string { func ignore(dir string, skip []string, fn func(string, os.FileInfo) error) func(string, os.FileInfo) error { return func(path string, info os.FileInfo) error { + if strings.HasSuffix(path, ".private.yml") || + strings.HasSuffix(path, ".private.yaml") { + return nil + } b, err := filepath.Rel(dir, path) if err != nil { return err diff --git a/tests/suites/static_analysis/lint_yaml.sh b/tests/suites/static_analysis/lint_yaml.sh index 3d51abe..51350e2 100644 --- a/tests/suites/static_analysis/lint_yaml.sh +++ b/tests/suites/static_analysis/lint_yaml.sh @@ -42,6 +42,7 @@ jobs: ignore: - build-dqlite - build-musl + - ci-trigger-github-com-juju-juju - ci-build-juju - ci-gating-tests - ci-proving-ground-tests @@ -78,7 +79,6 @@ jobs: - gating-functional-tests-ppc64el - test-refresh-multijob - build-jujud-operator-test - # TODO (stickupkid): The followng jobs seem to be orphan jobs with in the # jenkins suite. We should clean them up to ensure that they do run, or # are removed.