diff --git a/JenkinsJobs/Releng/publishToMaven.jenkinsfile b/JenkinsJobs/Releng/publishToMaven.jenkinsfile index 1d499c1deaa..f7873ec153c 100644 --- a/JenkinsJobs/Releng/publishToMaven.jenkinsfile +++ b/JenkinsJobs/Releng/publishToMaven.jenkinsfile @@ -14,31 +14,113 @@ pipeline { maven 'apache-maven-latest' } environment { + // Filter out all the feature, test, and product IUs that are not published. + EXCLUDED_ARTIFACTS = '\\.feature\\.group$|\\.feature\\.jar$|\\.test|\\.executable|\\.platform\\.ide|\\.platform\\.sdk|_root|\\.id$|\\.sdk\\.ide$' REPO = "${WORKSPACE}/repo" PATH = "${installMavenDaemon('1.0.2')}/bin:${PATH}" // Folder ~/.m2 is not writable for builds, ensure mvnd metadata are written within the workspace. // prevent jline warning about inability to create a system terminal and increase keep-alive timeouts to increase stability in concurrent usage MVND = "mvnd -Dmvnd.daemonStorage=${WORKSPACE}/tools/mvnd -Dorg.jline.terminal.type=dumb -Dmvnd.keepAlive=1000 -Dmvnd.maxLostKeepAlive=100" + ECLIPSE = "${installLatestEclipse()}" + URL_AGG_UPDATES = 'https://download.eclipse.org/cbi/updates/p2-aggregator/products/nightly/latest' } // parameters declared in the definition of the invoking job stages { stage('Aggregate Maven repository') { steps { - sh ''' - SCRIPT='git-repo/eclipse.platform.releng/publish-to-maven-central/CBIaggregator.sh' - chmod +x ${SCRIPT} - ${SCRIPT} ${snapshotOrRelease} + sh '''#!/bin/bash -e + echo "==== Install and run the CBI aggregator ====" + + DIR_AGGREGATOR="${WORKSPACE}/tools/aggregator" + MAVEN_PUBLISH_BASE="${WORKSPACE}/git-repo/eclipse.platform.releng/publish-to-maven-central" + FILE_SDK_AGGR="${MAVEN_PUBLISH_BASE}/SDK4Mvn.aggr" + + # Set whether this is a snapshot build or not + if [ "${snapshotOrRelease}" = "-snapshot" ]; then + sed -e 's/snapshot=".*"/snapshot="true"/g' -i ${FILE_SDK_AGGR} + fi + + echo "Installing the CBI aggregator into ${DIR_AGGREGATOR} ..." + ${ECLIPSE} --launcher.suppressErrors -noSplash \\ + -application org.eclipse.equinox.p2.director \\ + -r ${URL_AGG_UPDATES} \\ + -d ${DIR_AGGREGATOR} -p CBIProfile \\ + -installIU org.eclipse.cbi.p2repo.cli.product + + repoRaw="${WORKSPACE}/repo-raw" + mkdir ${repoRaw} + + echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..." + "${DIR_AGGREGATOR}/cbiAggr" aggregate --buildModel ${FILE_SDK_AGGR} \\ + --action CLEAN_BUILD --buildRoot ${repoRaw} \\ + -vmargs -Dorg.eclipse.ecf.provider.filetransfer.excludeContributors=org.eclipse.ecf.provider.filetransfer.httpclientjava + + mv ${repoRaw}/final ${REPO} + rm -rf ${repoRaw} + + echo "==== Enrich POMs ====" + # Add some required information to the generated poms: + # - dynamic content (retrieved mostly from MANIFEST.MF): + # - name + # - url + # - scm connection, tag and url + # - semi dynamic + # - developers (based on static map git-repo-base -> project leads) + # - static content + # - license + # - organization + # - issue management + + ENRICH_POMS_JAR=${WORKSPACE}/work/EnrichPoms.jar + ENRICH_POMS_PACKAGE=org.eclipse.platform.releng.maven.pom + + # build the jar: + mkdir -p ${WORKSPACE}/work/bin + javac -d "${WORKSPACE}/work/bin" $(find "${MAVEN_PUBLISH_BASE}/src" -name \\*.java) + pushd "${WORKSPACE}/work/bin" + jar --create --verbose --main-class=${ENRICH_POMS_PACKAGE}.EnrichPoms --file=${ENRICH_POMS_JAR} $(find * -name \\*.class) + popd + ls -l ${ENRICH_POMS_JAR} + + for project in {platform,jdt,pde}; do + echo "${project}" + java -jar ${ENRICH_POMS_JAR} "${REPO}/org/eclipse/${project}" + done + + echo "==== Add Javadoc stubs ====" + + # (groupSimpleName, javadocArtifactGA) + function createJavadocs() { + group=${1} + jar="${1}-javadoc.jar" + artifact=${2} + if [ -r ${jar} ]; then + rm ${jar} + fi + echo -e "Corresponding javadoc can be found in artifact ${artifact}\\n" > README.txt + jar cf ${jar} README.txt + for pom in org/eclipse/${group}/*/*/*.pom; do + pomFolder=$(dirname ${pom}) + if [[ ! $pomFolder =~ ${EXCLUDED_ARTIFACTS_PATTERN} ]]; then + javadoc=`echo ${pom} | sed -e "s|\\(.*\\)\\.pom|\\1-javadoc.jar|"` + cp ${jar} ${javadoc} + fi + done + } + + pushd ${REPO} + createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv + createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv + createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user + + # Because the pom enhancer modified the poms the checksums are wrong which produces noisy warnings. + # So regenerate the sha1 for every pom. + for i in $(find org -name *.pom); do + echo "Recalculate checksum of $i" + sha1sum -b < $i | awk '{print $1}' > $i.sha1 + done + popd ''' - dir("${REPO}") { - sh '''#!/bin/sh -e - # Because the pom enhancer modified the poms the checksums are wrong which produces noisy warnings. - # So regenerate the sha1 for every pom. - for i in $(find org -name *.pom); do - echo "Recalculate checksum of $i" - sha1sum -b < $i | awk '{print $1}' > $i.sha1 - done - ''' - } } } stage('Validate repository') { @@ -53,9 +135,12 @@ pipeline { projects='org/eclipse/pde org/eclipse/jdt/ org/eclipse/platform/' paths=$(find ${projects} -regextype posix-egrep -regex '.*/[0-9]+\\.[0-9]+[^/]*') for path in $paths; do - if [[ $path =~ \\.feature\\.group|\\.feature\\.jar|\\.executable|\\.test|\\.platform\\.ide|\\.platform\\.sdk|_root|\\.id/|\\.sdk\\.ide/ ]]; then - # Filter out all the feature, test, and product IUs that are not published. - continue + artifactId=$(basename $(dirname ${path})) + #TODO: Remove printout + echo "path: $path" + echo "artifactId: $artifactId" + if [[ $artifactId =~ ${EXCLUDED_ARTIFACTS} ]]; then + continue # Skip excluded artifact fi # And transform each path to a Maven artifact coordinate groupId:artifactId:version. elements=($(echo $path | tr '/' ' ')) #split by slash @@ -105,6 +190,10 @@ pipeline { set +x pomFolder=$(dirname ${pomFile#${REPO}/}) # name of folder, with leading REPO path stripped of version=$(basename ${pomFolder}) + artifactId=$(basename $(dirname "${pomFolder}")) + if [[ $artifactId =~ ${EXCLUDED_ARTIFACTS} ]]; then + continue # Skip excluded artifact + fi if [[ $version == *-SNAPSHOT ]]; then URL=https://repo.eclipse.org/content/repositories/eclipse-snapshots/ REPO_ID=repo.eclipse.org # server-id in the settings.xml, used for authentication @@ -128,6 +217,7 @@ pipeline { sourcesFile=$(echo "${pomFile}" | sed -e "s|\\(.*\\)\\.pom|\\1-sources.jar|") javadocFile=$(echo "${pomFile}" | sed -e "s|\\(.*\\)\\.pom|\\1-javadoc.jar|") echo "${file}" + echo "${artifactId}">>"coordinates.txt" if [ -f "${sourcesFile}" ]; then echo "${sourcesFile}" @@ -170,7 +260,7 @@ pipeline { always { archiveArtifacts allowEmptyArchive: true, artifacts: '\ repo/**, \ - repo-validation/coordinates.txt' + repo-validation/coordinates.txt, publish-*/coordinates.txt' } unsuccessful { emailext subject: "Publication of Maven artifacts failed", @@ -180,10 +270,23 @@ pipeline { } } -def installMavenDaemon(String version){ +def installMavenDaemon(String version) { return install('mvnd', "https://downloads.apache.org/maven/mvnd/${version}/maven-mvnd-${version}-linux-amd64.tar.gz") } +def installLatestEclipse(){ + def buildProps = null + dir("${WORKSPACE}/git-repo") { + def iBuild = sh(script: 'git describe --tags --match "I[0-9]*-[0-9]*" --abbrev=0', returnStdout: true).trim() + def buildpropertiesContent = sh(script: "curl -L https://download.eclipse.org/eclipse/downloads/drops4/${iBuild}/buildproperties.properties", returnStdout: true).trim() + buildProps = readProperties(text: buildpropertiesContent).collectEntries{n, v -> + v = v.trim(); + return [n, (v.startsWith('"') && v.endsWith('"') ? v.substring(1, v.length() - 1) : v)] + } + } + return install('eclipse', "https://download.eclipse.org/eclipse/downloads/drops4/${buildProps.PREVIOUS_RELEASE_ID}/eclipse-SDK-${buildProps.PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz") + '/eclipse' +} + def install(String toolType, String url) { dir("${WORKSPACE}/tools/${toolType}") { sh "curl -L ${url} | tar -xzf -" diff --git a/RELEASE.md b/RELEASE.md index 62268628734..725595614a5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -108,9 +108,8 @@ The actual steps to release - You can subscribe to [cross-project-issues](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev) to get the notifications on Simrel releases. * #### **Publish to Maven central** - Publishing to maven should happen by at least Tuesday before the release since there is up to a 24 hour delay for the maven mirrors. - - Update [SDK4Mvn.aggr](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/SDK4Mvn.aggr) and [CBIaggregator.sh](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/CBIaggregator.sh) to the release build. + - Update [SDK4Mvn.aggr](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/SDK4Mvn.aggr) to the release build. - SDK4Mvn.aggr determines what is being published to Maven - - Updating `CBIaggregator.sh` determines which Eclipse version is used to run the CBI aggregator - Run the [Publish to Maven](https://ci.eclipse.org/releng/job/Releng/job/PublishToMaven/) job in jenkins with the `-release` parameter. - Once that publish job has completed successfully, log into https://oss.sonatype.org/#stagingRepositories and close the Platform, JDT and PDE repositories. - If you do not have an account on oss.sonatype.org for performing the rest of the release request one by creating an issue like https://issues.sonatype.org/browse/OSSRH-43870 to get permissions for platform, JDT and PDE projects and tag an existing release engineer to give approval. diff --git a/eclipse.platform.releng/publish-to-maven-central/CBIaggregator.sh b/eclipse.platform.releng/publish-to-maven-central/CBIaggregator.sh deleted file mode 100755 index e46423db164..00000000000 --- a/eclipse.platform.releng/publish-to-maven-central/CBIaggregator.sh +++ /dev/null @@ -1,339 +0,0 @@ -#!/bin/bash -e -#******************************************************************************* -# Copyright (c) 2016, 2025 GK Software SE and others. -# -# This program and the accompanying materials -# are made available under the terms of the Eclipse Public License 2.0 -# which accompanies this distribution, and is available at -# https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# Stephan Herrmann - initial API and implementation -#******************************************************************************** - -#================================================================================ -# Parameters -#================================================================================ - -WORKING_ROOT=$(pwd) -# Directory containing this and other scripts and resources -BASE_DIR=$( cd -- "$(dirname ${0})" &> /dev/null && pwd ) - -# ECLIPSE: -SDK_BUILD_DIR=R-4.34-202411201800 -SDK_VERSION=4.34 -FILE_ECLIPSE="https://download.eclipse.org/eclipse/downloads/drops4/${SDK_BUILD_DIR}/eclipse-SDK-${SDK_VERSION}-linux-gtk-x86_64.tar.gz" - -# AGGREGATOR: -URL_AGG_UPDATES=https://download.eclipse.org/cbi/updates/p2-aggregator/products/nightly/latest - -# LOCAL TOOLS: -LOCAL_TOOLS=${WORKING_ROOT}/tools -DIR_AGGREGATOR=aggregator -AGGREGATOR=${LOCAL_TOOLS}/${DIR_AGGREGATOR}/cbiAggr -ECLIPSE=${LOCAL_TOOLS}/eclipse/eclipse - -# ENRICH POMS tool: -ENRICH_POMS_JAR=${WORKING_ROOT}/work/EnrichPoms.jar -ENRICH_POMS_PACKAGE=org.eclipse.platform.releng.maven.pom - -# AGGREGATION MODEL: -FILE_SDK_AGGR="${BASE_DIR}/SDK4Mvn.aggr" - -#================================================================================ -# Util functions -#================================================================================ -function require_executable() { - if [ -x ${1} ] - then - echo "Successfully installed: ${1}" - else - echo "not executable: ${1}" - ls -l ${1} - exit 1 - fi -} - -#================================================================================ -# (1) Install and run the CBI aggregator -#================================================================================ -echo "==== CBI aggregator ====" - -# Set whether this is a snapshot build or not -snapshot="false" -for arg in "$@"; do - echo $arg - if [ "$arg" = "-snapshot" ]; then - snapshot="true" - fi -done -sed -e "s/snapshot=\".*\"/snapshot=\"${snapshot}\"/g" -i ${FILE_SDK_AGGR} - - -if [ ! -d ${LOCAL_TOOLS} ] -then - mkdir ${LOCAL_TOOLS} -fi - -if [ ! -x ${ECLIPSE} ] -then - pushd ${LOCAL_TOOLS} - echo "Extracting Eclipse from ${FILE_ECLIPSE} ..." - curl -L ${FILE_ECLIPSE} | tar -xzf - - popd -fi -require_executable ${ECLIPSE} - -if [ ! -x ${AGGREGATOR} ] -then - echo "Installing the CBI aggregator into ${LOCAL_TOOLS}/${DIR_AGGREGATOR} ..." - ${ECLIPSE} --launcher.suppressErrors -noSplash \ - -application org.eclipse.equinox.p2.director \ - -r ${URL_AGG_UPDATES} \ - -d ${LOCAL_TOOLS}/${DIR_AGGREGATOR} -p CBIProfile \ - -installIU org.eclipse.cbi.p2repo.cli.product -fi -require_executable ${AGGREGATOR} - -RepoRaw="${WORKING_ROOT}/repo-raw" -Repo="${WORKING_ROOT}/repo" -mkdir ${RepoRaw} - -echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..." -${AGGREGATOR} aggregate --buildModel ${FILE_SDK_AGGR} \ - --action CLEAN_BUILD --buildRoot ${RepoRaw} \ - -vmargs -Dorg.eclipse.ecf.provider.filetransfer.excludeContributors=org.eclipse.ecf.provider.filetransfer.httpclientjava - -mv ${RepoRaw}/final ${Repo} -rm -rf ${RepoRaw} - -echo "========== Repo created: ==========" -du -sc ${Repo}/* -du -sc ${Repo}/org/* -du -sc ${Repo}/org/eclipse/* -echo "===================================" - - -#================================================================================ -# (2) Remove irrelevant stuff -#================================================================================ -# Removes from the build output of cbiAggregator everything that is not relevant for maven. -# All removed directories / files will be logged to .logs/removed.txt - -echo "==== Remove irrelevant stuff ====" - -pushd ${Repo} - -if [ ! -d .logs ] -then - mkdir .logs -elif [ -f .logs/removed.txt ] -then - rm .logs/removed.txt -fi - -#==== remove the p2 repository (not logged): ==== - -rm -rf p2.index p2.packed content.jar artifacts.jar - -#==== remove features: ==== - -echo "== Features: ==" | tee >> .logs/removed.txt - -find * -type d -name \*feature.group -print -exec rm -rf {} \; -prune >> .logs/removed.txt -find * -type d -name \*feature.jar -print -exec rm -rf {} \; -prune >> .logs/removed.txt - -#==== remove eclipse test plug-ins: ==== - -echo "== Test plugins: ==" | tee >> .logs/removed.txt - -ls -d org/eclipse/*/*.test* >> .logs/removed.txt -rm -r org/eclipse/*/*.test* - -#==== remove other non-artifacts: ==== - -echo "== Other non-artifacts: ==" | tee >> .logs/removed.txt - -find tooling -type d >> .logs/removed.txt -rm -r tooling* - -# ... folders that contain only 1.2.3/foo-1.2.3.pom but no corresponding 1.2.3/foo-1.2.3.jar: -function hasPomButNoJar() { - pushd ${1} - # expect only one sub-directory, starting with a digit, plus maven-metadata.xml*: - other=`ls -d [!0-9]* 2> /dev/null` - if echo "${other}" | tr "\n" " " | egrep "^maven-metadata.xml maven-metadata.xml.md5 maven-metadata.xml.sha1 \$" - then - : # clean -> proceed below - else - exit 1 # unexpected content found, don't remove - fi - # scan all *.pom inside the version sub-directory - r=1 - for pom in `ls [0-9]*/*.pom 2> /dev/null` - do - jar=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1.jar|"` - if [ -f ${jar} ] - then - # jar found, so keep it - exit 1 - fi - # pom without jar found, let's answer true below - r=0 - done - popd - exit $r -} -export -f hasPomButNoJar - -find org/eclipse/{jdt,pde,platform} -type d \ - -exec bash -c 'hasPomButNoJar "$@"' bash {} \; \ - -print -exec rm -rf {} \; -prune >> .logs/removed.txt -# second "bash" is used as $0 in the function - -popd - -echo "========== Repo reduced: ==========" -du -sc ${Repo}/* -du -sc ${Repo}/org/* -du -sc ${Repo}/org/eclipse/* -echo "===================================" - -#================================================================================ -# (3) Garbage Collector -#================================================================================ -# Removes from the build output of cbiAggregator everything that is not referenced -# from any pom below org/eclipse/{platform,jdt,pde} -# -# Log output: -# .logs/removedGarbage.txt all directories during garbage collection -# .logs/gc.log incoming dependencies of retained artifacts -# .logs/empty-dirs.txt removed empty directories - -echo "==== Garbage Collector ====" - -pushd ${Repo} - -#==== function gc_bundle(): ==== -# Test if pom ${1} is referenced in any other pom. -# If not, append the containing directory to the file "toremove.txt" -function gc_bundle { - AID=`echo ${1} | sed -e "s|.*/\(.*\)[_-].*|\1|"` - DIR=`echo ${1} | sed -e "s|\(.*\)/[0-9].*|\1|"` - POM=`basename ${1}` - - ANSWER=`find org/eclipse/{platform,jdt,pde} -name \*.pom \! -name ${POM} \ - -exec grep -q "${AID}" {} \; -print -quit` - - if [ "$ANSWER" == "" ] - then - echo "Will remove $DIR" - echo $DIR >> toremove.txt - else - echo "$1 is used by $ANSWER" - fi -} -export -f gc_bundle - -#==== run the garbage collector: ==== -# iterate (max 5 times) in case artifacts were used only from garbage: -for iteration in 1 2 3 4 5 -do - echo "== GC iteration ${iteration} ==" - - # look for garbage only outside platform, jdt or pde folders: - find -name platform -prune -o -name jdt -prune -o -name pde -prune -o \ - -name \*.pom -exec bash -c 'gc_bundle "$@"' bash {} \; \ - > gc-${iteration}.log - # second "bash" is used as $0 in the function - - if [ ! -f toremove.txt ] - then - # no more garbage found - break - fi - cat toremove.txt >> .logs/removedGarbage.txt - for d in `cat toremove.txt`; do rm -r $d; done - rm toremove.txt -done - -# merge gc logs: -cat gc-*.log | sort --unique > .logs/gc.log -rm gc-*.log - -#==== remove all directories that have become empty: ==== -for iteration in 1 2 3 4 5 6 ; do find -type d -empty -print \ - -exec rmdir {} \; -prune; done \ - >> .logs/empty-dirs.txt - -echo "========== Repo reduced: ==========" -du -sc ${Repo}/* -du -sc ${Repo}/org/* -du -sc ${Repo}/org/eclipse/* -echo "===================================" - -popd - -#================================================================================ -# (4) Enrich POMs -#================================================================================ -# Add some required information to the generated poms: -# - dynamic content (retrieved mostly from MANIFEST.MF): -# - name -# - url -# - scm connection, tag and url -# - semi dynamic -# - developers (based on static map git-repo-base -> project leads) -# - static content -# - license -# - organization -# - issue management - - -echo "==== Enrich POMs ====" - -# build the jar: -mkdir -p ${WORKING_ROOT}/work/bin -javac -d "${WORKING_ROOT}/work/bin" $(find "${BASE_DIR}/src" -name \*.java) -pushd "${WORKING_ROOT}/work/bin" -jar --create --verbose --main-class=${ENRICH_POMS_PACKAGE}.EnrichPoms --file=${ENRICH_POMS_JAR} $(find * -name \*.class) -popd -ls -l ${ENRICH_POMS_JAR} - -echo "platform" -java -jar ${ENRICH_POMS_JAR} "${Repo}/org/eclipse/platform" -echo "jdt" -java -jar ${ENRICH_POMS_JAR} "${Repo}/org/eclipse/jdt" -echo "pde" -java -jar ${ENRICH_POMS_JAR} "${Repo}/org/eclipse/pde" - - -echo "==== Add Javadoc stubs ====" - -# (groupSimpleName, javadocArtifactGA) -function createJavadocs() { - group=${1} - jar="${1}-javadoc.jar" - artifact=${2} - if [ -r ${jar} ] - then - rm ${jar} - fi - echo -e "Corresponding javadoc can be found in artifact ${artifact}\n" > README.txt - jar cf ${jar} README.txt - for pom in org/eclipse/${group}/*/*/*.pom - do - javadoc=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1-javadoc.jar|"` - cp ${jar} ${javadoc} - done -} - -pushd ${Repo} -createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv -createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv -createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user -popd - -echo "========== Repo completed ========="