[Build] Simplify exclusion of artifacts during Maven publication and inline CBIaggregator script

JenkinsJobs/Releng/publishToMaven.jenkinsfile

@@ -14,31 +14,113 @@ pipeline {
 		maven 'apache-maven-latest'
 	}
 	environment {
+		// Filter out all the feature, test, and product IUs that are not published.	
+		EXCLUDED_ARTIFACTS = '\\.feature\\.group$|\\.feature\\.jar$|\\.test|\\.executable|\\.platform\\.ide|\\.platform\\.sdk|_root|\\.id$|\\.sdk\\.ide$'
 		REPO = "${WORKSPACE}/repo"
 		PATH = "${installMavenDaemon('1.0.2')}/bin:${PATH}"
 		// Folder ~/.m2 is not writable for builds, ensure mvnd metadata are written within the workspace.
 		// prevent jline warning about inability to create a system terminal and increase keep-alive timeouts to increase stability in concurrent usage
 		MVND = "mvnd -Dmvnd.daemonStorage=${WORKSPACE}/tools/mvnd -Dorg.jline.terminal.type=dumb -Dmvnd.keepAlive=1000 -Dmvnd.maxLostKeepAlive=100"
+		ECLIPSE = "${installLatestEclipse()}"
+		URL_AGG_UPDATES = 'https://download.eclipse.org/cbi/updates/p2-aggregator/products/nightly/latest'
 	}
 	// parameters declared in the definition of the invoking job
 	stages {
 		stage('Aggregate Maven repository') {
 			steps {
-				sh '''
-					SCRIPT='git-repo/eclipse.platform.releng/publish-to-maven-central/CBIaggregator.sh'
-					chmod +x ${SCRIPT}
-					${SCRIPT} ${snapshotOrRelease}
+				sh '''#!/bin/bash -e
+					echo "==== Install and run the CBI aggregator ===="
+
+					DIR_AGGREGATOR="${WORKSPACE}/tools/aggregator"
+					MAVEN_PUBLISH_BASE="${WORKSPACE}/git-repo/eclipse.platform.releng/publish-to-maven-central"
+					FILE_SDK_AGGR="${MAVEN_PUBLISH_BASE}/SDK4Mvn.aggr"
+
+					# Set whether this is a snapshot build or not
+					if [ "${snapshotOrRelease}" = "-snapshot" ]; then
+						sed -e 's/snapshot=".*"/snapshot="true"/g' -i ${FILE_SDK_AGGR}
+					fi
+
+					echo "Installing the CBI aggregator into ${DIR_AGGREGATOR} ..."
+					${ECLIPSE} --launcher.suppressErrors -noSplash \\
+						-application org.eclipse.equinox.p2.director \\
+						-r ${URL_AGG_UPDATES} \\
+						-d ${DIR_AGGREGATOR} -p CBIProfile \\
+						-installIU org.eclipse.cbi.p2repo.cli.product
+
+					repoRaw="${WORKSPACE}/repo-raw"
+					mkdir ${repoRaw}
+
+					echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..."
+					"${DIR_AGGREGATOR}/cbiAggr" aggregate --buildModel ${FILE_SDK_AGGR} \\
+					  --action CLEAN_BUILD --buildRoot ${repoRaw} \\
+					  -vmargs -Dorg.eclipse.ecf.provider.filetransfer.excludeContributors=org.eclipse.ecf.provider.filetransfer.httpclientjava
+
+					mv ${repoRaw}/final ${REPO}
+					rm -rf ${repoRaw}
+
+					echo "==== Enrich POMs ===="
+					# Add some required information to the generated poms:
+					# - dynamic content (retrieved mostly from MANIFEST.MF):
+					#   - name
+					#   - url
+					#   - scm connection, tag and url
+					# - semi dynamic
+					#   - developers (based on static map git-repo-base -> project leads)
+					# - static content
+					#   - license
+					#   - organization
+					#   - issue management
+
+					ENRICH_POMS_JAR=${WORKSPACE}/work/EnrichPoms.jar
+					ENRICH_POMS_PACKAGE=org.eclipse.platform.releng.maven.pom
+
+					# build the jar:
+					mkdir -p ${WORKSPACE}/work/bin
+					javac -d "${WORKSPACE}/work/bin" $(find "${MAVEN_PUBLISH_BASE}/src" -name \\*.java)
+					pushd "${WORKSPACE}/work/bin"
+					jar --create --verbose --main-class=${ENRICH_POMS_PACKAGE}.EnrichPoms --file=${ENRICH_POMS_JAR} $(find * -name \\*.class)
+					popd
+					ls -l ${ENRICH_POMS_JAR}
+
+					for project in {platform,jdt,pde}; do
+					  echo "${project}"
+					  java -jar ${ENRICH_POMS_JAR} "${REPO}/org/eclipse/${project}"
+					done
+
+					echo "==== Add Javadoc stubs ===="
+
+					# (groupSimpleName, javadocArtifactGA)
+					function createJavadocs() {
+						group=${1}
+						jar="${1}-javadoc.jar"
+						artifact=${2}
+						if [ -r ${jar} ]; then
+							rm ${jar}
+						fi
+						echo -e "Corresponding javadoc can be found in artifact ${artifact}\\n" > README.txt
+						jar cf ${jar} README.txt
+						for pom in org/eclipse/${group}/*/*/*.pom; do
+							pomFolder=$(dirname ${pom})
+							if [[ ! $pomFolder =~ ${EXCLUDED_ARTIFACTS_PATTERN} ]]; then
+								javadoc=`echo ${pom} | sed -e "s|\\(.*\\)\\.pom|\\1-javadoc.jar|"`
+								cp ${jar} ${javadoc}
+							fi
+						done	
+					}
+
+					pushd ${REPO}
+					createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv
+					createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv
+					createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user
+
+					# Because the pom enhancer modified the poms the checksums are wrong which produces noisy warnings.
+					# So regenerate the sha1 for every pom.
+					for i in $(find org -name *.pom); do
+						echo "Recalculate checksum of $i"
+						sha1sum -b < $i | awk '{print $1}' > $i.sha1
+					done
+					popd
 				'''
-				dir("${REPO}") {
-					sh '''#!/bin/sh -e
-						# Because the pom enhancer modified the poms the checksums are wrong which produces noisy warnings.
-						# So regenerate the sha1 for every pom.
-						for i in $(find org -name *.pom); do
-							echo "Recalculate checksum of $i"
-							sha1sum -b < $i | awk '{print $1}' > $i.sha1
-						done
-					'''
-				}
 			}
 		}
 		stage('Validate repository') {
@@ -53,9 +135,12 @@ pipeline {
 						projects='org/eclipse/pde org/eclipse/jdt/ org/eclipse/platform/'
 						paths=$(find ${projects} -regextype posix-egrep -regex '.*/[0-9]+\\.[0-9]+[^/]*')
 						for path in $paths; do
-							if [[ $path =~ \\.feature\\.group|\\.feature\\.jar|\\.executable|\\.test|\\.platform\\.ide|\\.platform\\.sdk|_root|\\.id/|\\.sdk\\.ide/ ]]; then
-								# Filter out all the feature, test, and product IUs that are not published.
-								continue
+							artifactId=$(basename $(dirname ${path}))
+							#TODO: Remove printout
+							echo "path: $path"
+							echo "artifactId: $artifactId"
+							if [[ $artifactId =~ ${EXCLUDED_ARTIFACTS} ]]; then
+								continue # Skip excluded artifact
 							fi
 							# And transform each path to a Maven artifact coordinate groupId:artifactId:version.
 							elements=($(echo $path | tr '/' ' ')) #split by slash
@@ -105,6 +190,10 @@ pipeline {
 										set +x
 										pomFolder=$(dirname ${pomFile#${REPO}/}) # name of folder, with leading REPO path stripped of
 										version=$(basename ${pomFolder})
+										artifactId=$(basename $(dirname "${pomFolder}"))
+										if [[ $artifactId =~ ${EXCLUDED_ARTIFACTS} ]]; then
+											continue # Skip excluded artifact
+										fi
 										if [[ $version == *-SNAPSHOT ]]; then
 											URL=https://repo.eclipse.org/content/repositories/eclipse-snapshots/
 											REPO_ID=repo.eclipse.org # server-id in the settings.xml, used for authentication
@@ -128,6 +217,7 @@ pipeline {
 										sourcesFile=$(echo "${pomFile}" | sed -e "s|\\(.*\\)\\.pom|\\1-sources.jar|")
 										javadocFile=$(echo "${pomFile}" | sed -e "s|\\(.*\\)\\.pom|\\1-javadoc.jar|")
 										echo "${file}"
+										echo "${artifactId}">>"coordinates.txt"
 
 										if [ -f "${sourcesFile}" ]; then
 											echo "${sourcesFile}"
@@ -170,7 +260,7 @@ pipeline {
 		always {
 			archiveArtifacts allowEmptyArchive: true, artifacts: '\
 				repo/**, \
-				repo-validation/coordinates.txt'
+				repo-validation/coordinates.txt, publish-*/coordinates.txt'
 		}
 		unsuccessful {
 			emailext subject: "Publication of Maven artifacts failed",
@@ -180,10 +270,23 @@ pipeline {
 	}
 }
 
-def installMavenDaemon(String version){
+def installMavenDaemon(String version) {
 	return install('mvnd', "https://downloads.apache.org/maven/mvnd/${version}/maven-mvnd-${version}-linux-amd64.tar.gz")
 }
 
+def installLatestEclipse(){
+	def buildProps = null
+	dir("${WORKSPACE}/git-repo") {
+		def iBuild = sh(script: 'git describe --tags --match "I[0-9]*-[0-9]*" --abbrev=0', returnStdout: true).trim()
+		def buildpropertiesContent = sh(script: "curl -L https://download.eclipse.org/eclipse/downloads/drops4/${iBuild}/buildproperties.properties", returnStdout: true).trim()
+		buildProps = readProperties(text: buildpropertiesContent).collectEntries{n, v ->
+				v = v.trim();
+				return [n, (v.startsWith('"') && v.endsWith('"') ? v.substring(1, v.length() - 1) : v)]
+			}
+	}
+	return install('eclipse', "https://download.eclipse.org/eclipse/downloads/drops4/${buildProps.PREVIOUS_RELEASE_ID}/eclipse-SDK-${buildProps.PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz") + '/eclipse'
+}
+
 def install(String toolType, String url) {
 	dir("${WORKSPACE}/tools/${toolType}") {
 		sh "curl -L ${url} | tar -xzf -"

RELEASE.md

@@ -108,9 +108,8 @@ The actual steps to release
     - You can subscribe to [cross-project-issues](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev) to get the notifications on Simrel releases.
   * #### **Publish to Maven central**
     - Publishing to maven should happen by at least Tuesday before the release since there is up to a 24 hour delay for the maven mirrors.
-    - Update [SDK4Mvn.aggr](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/SDK4Mvn.aggr) and [CBIaggregator.sh](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/CBIaggregator.sh) to the release build.
+    - Update [SDK4Mvn.aggr](https://github.com/eclipse-platform/eclipse.platform.releng/blob/master/publish-to-maven-central/SDK4Mvn.aggr) to the release build.
       - SDK4Mvn.aggr determines what is being published to Maven
-      - Updating `CBIaggregator.sh` determines which Eclipse version is used to run the CBI aggregator
     - Run the [Publish to Maven](https://ci.eclipse.org/releng/job/Releng/job/PublishToMaven/) job in jenkins with the `-release` parameter.
     - Once that publish job has completed successfully, log into https://oss.sonatype.org/#stagingRepositories and close the Platform, JDT and PDE repositories.
       - If you do not have an account on oss.sonatype.org for performing the rest of the release request one by creating an issue like https://issues.sonatype.org/browse/OSSRH-43870 to get permissions for platform, JDT and PDE projects and tag an existing release engineer to give approval.