Skip to content

Commit

Permalink
STORM-3409: Build shaded deps with dependencies, instead of as a loos…
Browse files Browse the repository at this point in the history 
…e module, never upload Storm artifacts to Travis build cache to ensure a clean slate for each build. Remove license plugin configuration until we can upgrade to 1.21, as it breaks clean builds.
  • Loading branch information
@srdo
srdo committed Jun 20, 2019
1 parent 5d8731e commit 43dee13
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 139 deletions.
4 changes: 4 additions & 0 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ env:
dist: trusty
sudo: required

before_cache:
# Never upload Storm artifacts to the build cache, ensuring a clean slate every time
- rm -rf $HOME/.m2/repository/org/apache/storm

language: java
jdk:
- oraclejdk8
Expand Down
7 changes: 3 additions & 4 deletions dev-tools/travis/travis-install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,16 @@ TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
pip install --user -r ${TRAVIS_SCRIPT_DIR}/requirements.txt
pip3 install --user -r ${TRAVIS_SCRIPT_DIR}/requirements.txt

cd ${STORM_SRC_ROOT_DIR}/storm-shaded-deps/
python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install-shade.txt" mvn clean install --batch-mode
python ${TRAVIS_SCRIPT_DIR}/save-logs.py "storm-shaded-deps/install-shade.txt" mvn clean install --batch-mode -pl storm-shaded-deps -am
BUILD_RET_VAL=$?
if [[ "$BUILD_RET_VAL" != "0" ]];
then
cat "install-shade.txt"
cat "storm-shaded-deps/install-shade.txt"
exit ${BUILD_RET_VAL}
fi

cd ${STORM_SRC_ROOT_DIR}
python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" mvn clean install -DskipTests -Pnative,examples,externals '-P!include-shaded-deps' --batch-mode
python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" mvn clean install -DskipTests -Pnative,examples,externals -pl '!storm-shaded-deps' --batch-mode
BUILD_RET_VAL=$?

if [[ "$BUILD_RET_VAL" != "0" ]];
Expand Down
6 changes: 3 additions & 3 deletions dev-tools/travis/travis-script.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,17 @@ elif [ "$2" == "External" ]
then
if [ "$TRAVIS_JDK_VERSION" == "openjdk11" ]
then
TEST_MODULES='!storm-client,!storm-server,!storm-core,!storm-webapp,!external/storm-cassandra,!external/storm-hive,!external/storm-hdfs,!external/storm-hbase,!sql/storm-sql-external/storm-sql-hdfs,!external/storm-hdfs-blobstore'
TEST_MODULES='!storm-client,!storm-server,!storm-core,!storm-webapp,!storm-shaded-deps,!external/storm-cassandra,!external/storm-hive,!external/storm-hdfs,!external/storm-hbase,!sql/storm-sql-external/storm-sql-hdfs,!external/storm-hdfs-blobstore'
else
TEST_MODULES='!storm-client,!storm-server,!storm-core,!storm-webapp'
TEST_MODULES='!storm-client,!storm-server,!storm-core,!storm-webapp,!storm-shaded-deps'
fi
fi
# We should be concerned that Travis CI could be very slow because it uses VM
export STORM_TEST_TIMEOUT_MS=150000
# Travis only has 3GB of memory, lets use 1GB for build, and 1.5GB for forked JVMs
export MAVEN_OPTS="-Xmx1024m"

mvn --batch-mode test -fae -Pnative,all-tests,examples,externals '-P!include-shaded-deps' -Prat -pl "$TEST_MODULES"
mvn --batch-mode test -fae -Pnative,all-tests,examples,externals -Prat -pl "$TEST_MODULES"
BUILD_RET_VAL=$?

for dir in `find . -type d -and -wholename \*/target/\*-reports`;
Expand Down
129 changes: 1 addition & 128 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,7 @@
</properties>

<modules>
<module>storm-shaded-deps</module>
<module>storm-checkstyle</module>
<module>storm-multilang/javascript</module>
<module>storm-multilang/python</module>
Expand Down Expand Up @@ -378,15 +379,6 @@
<provided.scope>compile</provided.scope>
</properties>
</profile>
<profile>
<id>include-shaded-deps</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<modules>
<module>storm-shaded-deps</module>
</modules>
</profile>
<profile>
<id>rat</id>
<build>
Expand Down Expand Up @@ -1313,100 +1305,6 @@
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
<version>1.19</version>
<configuration>
<useMissingFile>true</useMissingFile>
<failOnMissing>true</failOnMissing>
<includeTransitiveDependencies>true</includeTransitiveDependencies>
<fileTemplate>/org/codehaus/mojo/license/third-party-file-groupByMultiLicense.ftl</fileTemplate>
<excludedScopes>system,test</excludedScopes>
<licenseMerges>
<licenseMerge>
Apache License, Version 2.0 |
Apache License, version 2.0 |
Apache License Version 2 |
Apache License Version 2.0 |
Apache License version 2.0 |
Apache 2 |
Apache 2.0 |
Apache License, 2.0 |
Apache License 2 |
Apache License 2.0 |
Apache Public License 2.0 |
Apache Software License - Version 2.0 |
Apache v2 |
ASL, version 2 |
The Apache License, Version 2.0 |
The Apache Software License, Version 2.0
</licenseMerge>
<licenseMerge>
Apache License |
Apache Software Licenses
</licenseMerge>
<licenseMerge>
BSD License |
BSD license |
BSD |
The BSD License
</licenseMerge>
<licenseMerge>
BSD 3-Clause License |
BSD 3-Clause |
BSD 3-clause |
The BSD 3-Clause License |
New BSD License |
New BSD license
</licenseMerge>
<licenseMerge>
Common Development and Distribution License (CDDL) v1.0 |
COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 |
CDDL 1.0
</licenseMerge>
<licenseMerge>
Common Development and Distribution License (CDDL) v1.1 |
COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1 |
CDDL 1.1 |
Common Development and Distribution License (CDDL), Version 1.1
</licenseMerge>
<licenseMerge>
Common Development and Distribution License |
<!-- Multilicense, choosing CDDL -->
CDDL+GPL |
CDDL+GPL License |
CDDL + GPLv2 with classpath exception
</licenseMerge>
<licenseMerge>
Eclipse Public License, Version 1.0 |
Eclipse Public License 1.0 |
Eclipse Public License - v 1.0
</licenseMerge>
<licenseMerge>
MIT License |
The MIT License |
MIT license |
MIT X11 License |
MIT
</licenseMerge>
<licenseMerge>
The GNU General Public License (GPL), Version 2, With Classpath Exception |
GPL2 w/ CPE
</licenseMerge>
<licenseMerge>
GNU Lesser General Public License (LGPL), Version 2.1 |
LGPL, version 2.1 |
GNU Lesser General Public License Version 2.1 |
GNU Lesser General Public License, version 2.1
</licenseMerge>
<licenseMerge>
Common Public License Version 1.0 |
Common Public License - v 1.0
</licenseMerge>
</licenseMerges>
</configuration>
</plugin>
</plugins>
</pluginManagement>

Expand Down Expand Up @@ -1471,31 +1369,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<thirdPartyFilename>DEPENDENCY-LICENSES</thirdPartyFilename>
<outputDirectory>${project.basedir}</outputDirectory>
<missingFile>${project.basedir}/THIRD-PARTY.properties</missingFile>
<aggregateMissingLicensesFile>${project.basedir}/THIRD-PARTY.properties</aggregateMissingLicensesFile>
</configuration>
<executions>
<execution>
<id>generate-and-check-licenses</id>
<goals>
<goal>aggregate-add-third-party</goal>
</goals>
</execution>
<execution>
<id>download-licenses</id>
<goals>
<goal>aggregate-download-licenses</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

Expand Down
4 changes: 0 additions & 4 deletions storm-client/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,10 +240,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-pmd-plugin</artifactId>
Expand Down

0 comments on commit 43dee13

Please sign in to comment.