From a609b3a0133c71a5646ffbfe73300b98372e8d05 Mon Sep 17 00:00:00 2001 From: Yizhao Lang Date: Wed, 8 May 2024 12:34:09 -0400 Subject: [PATCH] SDEV-1117: Make sure target resolution always happens from within an MasterToSlaveCallable (#312) * SDEV-1117: Make sure target resolution always happens from within an MasterToSlaveCallable For callflow we will either reuse the scan patterns provied for the traditional scan or alternatively make use of an ovveride. In either case before hadning off to dependencies to do the actual work the patterns need to be resolved to file targets. Because jenkins manages a cluster this needs to be done from the right context. I have refactored out the actuall file resolution logic to make it easier to re-use and created a new class that extends MasterToSlaveCallable so that we can leverage the logic from the right place. The orginal place where this logic was used inside RemoteScanner was already within a MasterToSlaveCallable class, so nothin needed to change there, other than pointing to the logics new location. Some tests were refactored to reflect the new location of the logic * Add license header * make codenarc happy by hook or by crook * Fix a test * Add license header * Ci build fix attempt --------- Co-authored-by: Chris Wininger Co-authored-by: Eduard Tita --- pom.xml | 6 ++ .../nexus/ci/iq/IqPolicyEvaluatorUtil.groovy | 18 ++--- .../nexus/ci/iq/RemoteFileResolver.groovy | 35 ++++++++ .../sonatype/nexus/ci/iq/RemoteScanner.groovy | 21 +---- .../nexus/ci/iq/ScanPatternUtil.groovy | 39 +++++++++ .../nexus/ci/iq/IqPolicyEvaluatorTest.groovy | 14 ++-- .../nexus/ci/iq/RemoteScannerTest.groovy | 22 +---- .../nexus/ci/iq/ScanPatternUtilTest.groovy | 81 +++++++++++++++++++ 8 files changed, 178 insertions(+), 58 deletions(-) create mode 100644 src/main/java/org/sonatype/nexus/ci/iq/RemoteFileResolver.groovy create mode 100644 src/main/java/org/sonatype/nexus/ci/iq/ScanPatternUtil.groovy create mode 100644 src/test/java/org/sonatype/nexus/ci/iq/ScanPatternUtilTest.groovy diff --git a/pom.xml b/pom.xml index ce371ce4..8bbaa278 100644 --- a/pom.xml +++ b/pom.xml @@ -141,6 +141,12 @@ spring-web 5.3.28 + + + org.springframework.security + spring-security-web + 5.8.4 + org.apache.commons diff --git a/src/main/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorUtil.groovy b/src/main/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorUtil.groovy index dea3969f..afca645f 100644 --- a/src/main/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorUtil.groovy +++ b/src/main/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorUtil.groovy @@ -107,7 +107,8 @@ class IqPolicyEvaluatorUtil envVars) def repositoryUrl = launcher.getChannel().call(repositoryUrlFinder) if (repositoryUrl != null) { - def repositoryPath = Paths.get(workspace.getRemote(),".git").toString() + def repositoryPath = Paths.get(workspace.getRemote(), '.git').toString() + iqClient.addOrUpdateSourceControl(applicationId, repositoryUrl, repositoryPath) } @@ -121,8 +122,8 @@ class IqPolicyEvaluatorUtil CallflowConfiguration callflowConfiguration = iqPolicyEvaluator.getCallflowConfiguration() callflowOptions = makeCallflowOptions( + launcher, callflowConfiguration, - remoteScanner, workDirectory, envVars, iqPolicyEvaluator.iqScanPatterns @@ -212,18 +213,16 @@ class IqPolicyEvaluatorUtil } private static CallflowOptions makeCallflowOptions( + final Launcher launcher, final CallflowConfiguration callflowConfiguration, - final RemoteScanner remoteScanner, final File workdir, final EnvVars envVars, final List iqScanPatterns) { if (callflowConfiguration == null) { final List expandedPatterns = getScanPatterns(iqScanPatterns, envVars) - final List targets = remoteScanner.getScanTargets(workdir, expandedPatterns) - .collect { - return it.getAbsolutePath() - } + final RemoteFileResolver remoteFileResolver = new RemoteFileResolver(workdir, expandedPatterns) + final List targets = launcher.getChannel().call(remoteFileResolver) // defaults to using same targets as original iq scan, when enabled but no additional config passed return new CallflowOptions(targets, null, null) @@ -235,9 +234,8 @@ class IqPolicyEvaluatorUtil } final List expandedPatterns = getScanPatterns(patterns, envVars) - - final List targets = remoteScanner.getScanTargets(workdir, expandedPatterns) - .collect { it.getAbsolutePath() } + final RemoteFileResolver remoteFileResolver = new RemoteFileResolver(workdir, expandedPatterns) + final List targets = launcher.getChannel().call(remoteFileResolver) final Properties addtionalConfiguration = new Properties() if (callflowConfiguration.getAdditionalConfiguration() != null) { diff --git a/src/main/java/org/sonatype/nexus/ci/iq/RemoteFileResolver.groovy b/src/main/java/org/sonatype/nexus/ci/iq/RemoteFileResolver.groovy new file mode 100644 index 00000000..a71e4ef1 --- /dev/null +++ b/src/main/java/org/sonatype/nexus/ci/iq/RemoteFileResolver.groovy @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016-present Sonatype, Inc. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ + package org.sonatype.nexus.ci.iq + +import jenkins.security.MasterToSlaveCallable + +class RemoteFileResolver + extends MasterToSlaveCallable, RuntimeException> +{ + private final File workDir; + private final List scanPatterns; + + RemoteFileResolver(final File workDir, final List scanPatterns) { + this.workDir = workDir + this.scanPatterns = scanPatterns + } + + @Override + List call() throws RuntimeException { + return ScanPatternUtil.getScanTargets(workDir, scanPatterns) + .collect { + return it.getAbsolutePath() + } + } +} diff --git a/src/main/java/org/sonatype/nexus/ci/iq/RemoteScanner.groovy b/src/main/java/org/sonatype/nexus/ci/iq/RemoteScanner.groovy index e9620af6..afa51fdc 100644 --- a/src/main/java/org/sonatype/nexus/ci/iq/RemoteScanner.groovy +++ b/src/main/java/org/sonatype/nexus/ci/iq/RemoteScanner.groovy @@ -23,9 +23,6 @@ import org.slf4j.Logger class RemoteScanner extends MasterToSlaveCallable { - static final List DEFAULT_SCAN_PATTERN = - ['**/*.jar', '**/*.war', '**/*.ear', '**/*.zip', '**/*.tar.gz'] - static final List DEFAULT_MODULE_INCLUDES = ['**/sonatype-clm/module.xml', '**/nexus-iq/module.xml'] @@ -91,7 +88,7 @@ class RemoteScanner InternalIqClient iqClient = IqClientFactory.getIqLocalClient(log, instanceId) def workDirectory = new File(workspace.getRemote()) - def targets = getScanTargets(workDirectory, scanPatterns) + def targets = ScanPatternUtil.getScanTargets(workDirectory, scanPatterns) def filesExcludesSet = [] as Set for (String pattern : scanPatterns) { if (pattern.startsWith(CONTAINER)) { @@ -112,22 +109,6 @@ class RemoteScanner return new RemoteScanResult(scanResult.scan, new FilePath(scanResult.scanFile)) } - List getScanTargets(final File workDir, final List scanPatterns) { - def directoryScanner = RemoteScannerFactory.getDirectoryScanner() - def normalizedScanPatterns = scanPatterns ?: DEFAULT_SCAN_PATTERN - def includeScanPatterns = normalizedScanPatterns.findAll{!it.startsWith(EXCLUDE_MARKER)} - def excludeScanPatterns = normalizedScanPatterns.findAll{it.startsWith(EXCLUDE_MARKER)}.collect{it.substring(1)} - directoryScanner.setBasedir(workDir) - directoryScanner.setIncludes(includeScanPatterns.toArray(new String[includeScanPatterns.size()])) - directoryScanner.setExcludes(excludeScanPatterns.toArray(new String[excludeScanPatterns.size()])) - directoryScanner.addDefaultExcludes() - directoryScanner.scan() - return (directoryScanner.getIncludedDirectories() + directoryScanner.getIncludedFiles()) - .collect { f -> new File(workDir, f) } - .sort() - .asImmutable() - } - List getModuleIndices(final File workDirectory, final List moduleExcludes) { final DirectoryScanner directoryScanner = RemoteScannerFactory.getDirectoryScanner() directoryScanner.setBasedir(workDirectory) diff --git a/src/main/java/org/sonatype/nexus/ci/iq/ScanPatternUtil.groovy b/src/main/java/org/sonatype/nexus/ci/iq/ScanPatternUtil.groovy new file mode 100644 index 00000000..2bc29dbd --- /dev/null +++ b/src/main/java/org/sonatype/nexus/ci/iq/ScanPatternUtil.groovy @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016-present Sonatype, Inc. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package org.sonatype.nexus.ci.iq + +@SuppressWarnings(['AbcMetric']) +class ScanPatternUtil +{ + static final List DEFAULT_SCAN_PATTERN = + ['**/*.jar', '**/*.war', '**/*.ear', '**/*.zip', '**/*.tar.gz'] + + public static final String EXCLUDE_MARKER = '!' + + // Warning: Only use this in the context of a MasterToSlaveCallable + static List getScanTargets(final File workDir, final List scanPatterns) { + def directoryScanner = RemoteScannerFactory.getDirectoryScanner() + def normalizedScanPatterns = scanPatterns ?: DEFAULT_SCAN_PATTERN + def includeScanPatterns = normalizedScanPatterns.findAll{!it.startsWith(EXCLUDE_MARKER)} + def excludeScanPatterns = normalizedScanPatterns.findAll{it.startsWith(EXCLUDE_MARKER)}.collect{it.substring(1)} + directoryScanner.setBasedir(workDir) + directoryScanner.setIncludes(includeScanPatterns.toArray(new String[includeScanPatterns.size()])) + directoryScanner.setExcludes(excludeScanPatterns.toArray(new String[excludeScanPatterns.size()])) + directoryScanner.addDefaultExcludes() + directoryScanner.scan() + return (directoryScanner.getIncludedDirectories() + directoryScanner.getIncludedFiles()) + .collect { f -> new File(workDir, f) } + .sort() + .asImmutable() + } +} diff --git a/src/test/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorTest.groovy b/src/test/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorTest.groovy index 15bbbb80..1b488c2d 100644 --- a/src/test/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorTest.groovy +++ b/src/test/java/org/sonatype/nexus/ci/iq/IqPolicyEvaluatorTest.groovy @@ -716,14 +716,14 @@ class IqPolicyEvaluatorTest iqClient.verifyOrCreateApplication(*_) >> true iqClient.getProprietaryConfigForApplicationEvaluation('appId') >> proprietaryConfig RemoteScannerFactory.getRemoteScanner(*_) >> remoteScanner - channel.call(_) >> remoteScanResult + channel.call(remoteScanner) >> remoteScanResult when: getBuildStepForCallflowTests(true, null) .perform((AbstractBuild) run, launcher, Mock(BuildListener)) then: 'evaluates the results using default callflow options' - 1 * remoteScanner.getScanTargets(_, _) >> [pathReturnedByExpandingIqScanPatterns] + 1 * channel.call(_ as RemoteFileResolver) >> [pathReturnedByExpandingIqScanPatterns.getAbsolutePath()] 1 * iqClient.evaluateApplication('appId', 'stage', scanResult, _, { it.scanTargets == defaultCallflowOptions.scanTargets && it.namespaces == null && @@ -735,10 +735,8 @@ class IqPolicyEvaluatorTest setup: def expectedNamespaces = ['any.namespace'] def givenAdditionalConfig = [some: "property"] - def expectedScanTargets = [ - new File("some-path-1").getAbsolutePath(), - new File('some-path-2').getAbsolutePath() - ] + def expectedScanTargets = ["some-path-1", "some-path-2"] + def expectedProps = new Properties().with { it.put("some", "property") it @@ -750,7 +748,7 @@ class IqPolicyEvaluatorTest iqClient.verifyOrCreateApplication(*_) >> true iqClient.getProprietaryConfigForApplicationEvaluation('appId') >> proprietaryConfig RemoteScannerFactory.getRemoteScanner(*_) >> remoteScanner - channel.call(_) >> remoteScanResult + channel.call(remoteScanner) >> remoteScanResult when: getBuildStepForCallflowTests( @@ -762,7 +760,7 @@ class IqPolicyEvaluatorTest ).perform((AbstractBuild) run, launcher, Mock(BuildListener)) then: 'evaluates the results using default callflow options' - 1 * remoteScanner.getScanTargets(*_) >> [new File("some-path-1"), new File('some-path-2')] + 1 * channel.call(_ as RemoteFileResolver) >> ["some-path-1", 'some-path-2'] 1 * iqClient.evaluateApplication('appId', 'stage', scanResult, _, { it.scanTargets == expectedScanTargets && it.namespaces == expectedNamespaces && diff --git a/src/test/java/org/sonatype/nexus/ci/iq/RemoteScannerTest.groovy b/src/test/java/org/sonatype/nexus/ci/iq/RemoteScannerTest.groovy index 56f3b92d..09d3c54f 100644 --- a/src/test/java/org/sonatype/nexus/ci/iq/RemoteScannerTest.groovy +++ b/src/test/java/org/sonatype/nexus/ci/iq/RemoteScannerTest.groovy @@ -153,23 +153,6 @@ class RemoteScannerTest 'Base Directory' | 5 | new File('/file/path') } - def 'Uses default scan patterns when patterns not set'() { - setup: - def workspaceFile = new File('/file/path') - final RemoteScanner remoteScanner = new RemoteScanner('appId', 'stageId', [], [], new FilePath(workspaceFile), - proprietaryConfig, log, 'instanceId', null, null) - directoryScanner.getIncludedDirectories() >> [] - directoryScanner.getIncludedFiles() >> [] - - when: - remoteScanner.getScanTargets(workspaceFile, []) - - then: - 1 * directoryScanner.setIncludes(*_) >> { arguments -> - assert arguments[0] == RemoteScanner.DEFAULT_SCAN_PATTERN - } - } - def 'Uses default modules for includes'() { setup: def workspaceFile = new File('/file/path') @@ -205,7 +188,6 @@ class RemoteScannerTest } } - def 'Uses no excludes by default'() { setup: def workspaceFile = new File('/file/path') @@ -215,7 +197,7 @@ class RemoteScannerTest directoryScanner.getIncludedFiles() >> [] when: - remoteScanner.getScanTargets(workspaceFile, ['*.jar']) + ScanPatternUtil.getScanTargets(workspaceFile, ['*.jar']) then: 1 * directoryScanner.setIncludes(*_) >> { arguments -> @@ -236,7 +218,7 @@ class RemoteScannerTest directoryScanner.getIncludedFiles() >> [] when: - remoteScanner.getScanTargets(workspaceFile, ['*.jar','!*.zip','*.war','!*.tar']) + ScanPatternUtil.getScanTargets(workspaceFile, ['*.jar','!*.zip','*.war','!*.tar']) then: 1 * directoryScanner.setIncludes(*_) >> { arguments -> diff --git a/src/test/java/org/sonatype/nexus/ci/iq/ScanPatternUtilTest.groovy b/src/test/java/org/sonatype/nexus/ci/iq/ScanPatternUtilTest.groovy new file mode 100644 index 00000000..7b18f985 --- /dev/null +++ b/src/test/java/org/sonatype/nexus/ci/iq/ScanPatternUtilTest.groovy @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016-present Sonatype, Inc. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package org.sonatype.nexus.ci.iq + +import org.codehaus.plexus.util.DirectoryScanner +import spock.lang.Specification +import spock.util.mop.ConfineMetaClassChanges + +@ConfineMetaClassChanges([IqClientFactory]) +class ScanPatternUtilTest + extends Specification +{ + DirectoryScanner directoryScanner + + def setup() { + GroovyMock(RemoteScannerFactory, global: true) + directoryScanner = Mock() + RemoteScannerFactory.getDirectoryScanner() >> directoryScanner + } + + def 'Uses default scan patterns when patterns not set'() { + setup: + def workspaceFile = new File('/file/path') + directoryScanner.getIncludedDirectories() >> [] + directoryScanner.getIncludedFiles() >> [] + + when: + ScanPatternUtil.getScanTargets(workspaceFile, []) + + then: + 1 * directoryScanner.setIncludes(*_) >> { arguments -> + assert arguments[0] == ScanPatternUtil.DEFAULT_SCAN_PATTERN + } + } + + def 'Uses no excludes by default'() { + setup: + def workspaceFile = new File('/file/path') + directoryScanner.getIncludedDirectories() >> [] + directoryScanner.getIncludedFiles() >> [] + + when: + ScanPatternUtil.getScanTargets(workspaceFile, ['*.jar']) + + then: + 1 * directoryScanner.setIncludes(*_) >> { arguments -> + assert arguments[0] == ['*.jar'] + } + 1 * directoryScanner.setExcludes(*_) >> { arguments -> + assert arguments[0] == [] + } + } + + def 'Pass excludes to directory scanner'() { + setup: + def workspaceFile = new File('/file/path') + directoryScanner.getIncludedDirectories() >> [] + directoryScanner.getIncludedFiles() >> [] + + when: + ScanPatternUtil.getScanTargets(workspaceFile, ['*.jar','!*.zip','*.war','!*.tar']) + + then: + 1 * directoryScanner.setIncludes(*_) >> { arguments -> + assert arguments[0] == ['*.jar','*.war'] + } + 1 * directoryScanner.setExcludes(*_) >> { arguments -> + assert arguments[0] == ['*.zip','*.tar'] + } + } +}