Merge 'origin/main' into lucene_snapshot

ShourieG · Nov 29, 2023 · fa4057d · fa4057d
2 parents 8502d7a + 3b834b5
commit fa4057d
Show file tree

Hide file tree

Showing 178 changed files with 3,897 additions and 2,082 deletions.
diff --git a/.buildkite/scripts/dra-workflow.sh b/.buildkite/scripts/dra-workflow.sh
@@ -13,8 +13,8 @@ fi
 echo --- Preparing
 
 # TODO move this to image
-sudo apt-get update -y
-sudo apt-get install -y libxml2-utils python3.10-venv
+sudo NEEDRESTART_MODE=l apt-get update -y
+sudo NEEDRESTART_MODE=l apt-get install -y libxml2-utils python3.10-venv
 
 RM_BRANCH="$BRANCH"
 if [[ "$BRANCH" == "main" ]]; then

diff --git a/...c/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/...c/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java
@@ -176,6 +176,11 @@ public SearchLookup lookup() {
                 public Set<String> sourcePaths(String name) {
                     return Set.of(name);
                 }
+
+                @Override
+                public String parentField(String field) {
+                    throw new UnsupportedOperationException();
+                }
             });
         }
         throw new IllegalArgumentException("can't read [" + name + "]");

diff --git a/...c/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy b/...c/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy
@@ -31,7 +31,8 @@ abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {
         }
         """
 
-        subProject(":distribution:archives:integ-test-zip") << "configurations { extracted }"
+        subProject(":distribution:archives:integ-test-zip") << "configurations.create('extracted')\n"
+        subProject(":distribution:archives:integ-test-zip") << "configurations.create('default')\n"
     }
 
     void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackTests = []) {

diff --git a/...roovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy b/...roovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
@@ -24,8 +24,6 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTest does nothing when there are no tests"() {
         given:
-        // RestIntegTestTask not cc compatible due to
-        configurationCacheCompatible = false
         buildFile << """
         plugins {
           id 'elasticsearch.legacy-yaml-rest-test'
@@ -43,8 +41,6 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTest executes and copies api and tests to correct source set"() {
         given:
-        // RestIntegTestTask not cc compatible due to
-        configurationCacheCompatible = false
         internalBuild()
         buildFile << """
             apply plugin: 'elasticsearch.legacy-yaml-rest-test'
@@ -56,9 +52,10 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
             // can't actually spin up test cluster from this test
            tasks.withType(Test).configureEach{ enabled = false }
 
+           def clazzpath = sourceSets.yamlRestTest.runtimeClasspath
            tasks.register("printYamlRestTestClasspath").configure {
                doLast {
-                   println sourceSets.yamlRestTest.runtimeClasspath.asPath
+                   println clazzpath.asPath
                }
            }
         """

diff --git a/...ternal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/...ternal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java
@@ -74,13 +74,12 @@ public void doCall() {
                 }
             });
 
+        boolean isModule = GradleUtils.isModuleProject(project.getPath());
+        boolean isXPackModule = isModule && project.getPath().startsWith(":x-pack");
+        if (isModule == false || isXPackModule) {
+            addNoticeGeneration(project, extension);
+        }
         project.afterEvaluate(p -> {
-            boolean isModule = GradleUtils.isModuleProject(p.getPath());
-            boolean isXPackModule = isModule && p.getPath().startsWith(":x-pack");
-            if (isModule == false || isXPackModule) {
-                addNoticeGeneration(p, extension);
-            }
-
             @SuppressWarnings("unchecked")
             NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
                 .getExtensions()

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java
@@ -16,11 +16,11 @@
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.file.SourceDirectorySet;
 import org.gradle.api.internal.file.FileOperations;
-import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.PathSensitive;
@@ -43,7 +43,7 @@
  * A task to create a notice file which includes dependencies' notices.
  */
 @CacheableTask
-public class NoticeTask extends DefaultTask {
+public abstract class NoticeTask extends DefaultTask {
 
     @InputFile
     @PathSensitive(PathSensitivity.RELATIVE)
@@ -57,19 +57,17 @@ public class NoticeTask extends DefaultTask {
     /**
      * Directories to include notices from
      */
-    private final ListProperty<File> licensesDirs;
+    @Internal
+    abstract ListProperty<File> getLicenseDirs();
 
     private final FileOperations fileOperations;
-    private ObjectFactory objectFactory;
 
     @Inject
-    public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOperations fileOperations, ObjectFactory objectFactory) {
-        this.objectFactory = objectFactory;
+    public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOperations fileOperations) {
         this.fileOperations = fileOperations;
         setDescription("Create a notice file from dependencies");
         // Default licenses directory is ${projectDir}/licenses (if it exists)
-        licensesDirs = objectFactory.listProperty(File.class);
-        licensesDirs.add(projectLayout.getProjectDirectory().dir("licenses").getAsFile());
+        getLicenseDirs().add(projectLayout.getProjectDirectory().dir("licenses").getAsFile());
         inputFile = new File(buildLayout.getRootDirectory(), "NOTICE.txt");
         outputFile = projectLayout.getBuildDirectory().dir("notices/" + getName()).get().file("NOTICE.txt").getAsFile();
     }
@@ -78,7 +76,7 @@ public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOper
      * Add notices from the specified directory.
      */
     public void licensesDir(File licensesDir) {
-        licensesDirs.add(licensesDir);
+        getLicenseDirs().add(licensesDir);
     }
 
     public void source(Object source) {
@@ -185,7 +183,7 @@ public FileCollection getNoticeFiles() {
     }
 
     private List<File> existingLicenseDirs() {
-        return licensesDirs.get().stream().filter(d -> d.exists()).collect(Collectors.toList());
+        return getLicenseDirs().get().stream().filter(d -> d.exists()).collect(Collectors.toList());
     }
 
     @InputFiles

diff --git a/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -35,7 +35,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":modules:percolator");
         map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval");
         map.put(LegacyRestTestBasePlugin.class, ":modules:reindex");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:repository-s3");
         map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url");
         map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu");
@@ -138,11 +137,9 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:multi-cluster-tests-with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:native-multi-node-tests");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:single-node-tests");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:repositories-metering-api:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:minio");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:url");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:tls-basic");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:multi-node");
@@ -151,11 +148,9 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:fs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:minio");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:s3");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:multi-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:no-sql");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:single-node");

diff --git a/...ternal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java b/...ternal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
@@ -29,11 +29,8 @@
 import org.gradle.api.specs.NotSpec;
 import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.Sync;
-import org.gradle.api.tasks.TaskContainer;
 import org.gradle.api.tasks.bundling.Zip;
 
-import java.util.Collections;
-
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.RestrictedBuildApiService.BUILD_API_RESTRICTIONS_SYS_PROPERTY;
@@ -131,11 +128,7 @@ public void apply(Project project) {
     }
 
     private void configureCacheability(StandaloneRestIntegTestTask testTask) {
-        TaskContainer tasks = project.getTasks();
-        Spec<Task> taskSpec = t -> tasks.withType(StandaloneRestIntegTestTask.class)
-            .stream()
-            .filter(task -> task != testTask)
-            .anyMatch(task -> Collections.disjoint(task.getClusters(), testTask.getClusters()) == false);
+        Spec<Task> taskSpec = task -> testTask.getClusters().stream().anyMatch(ElasticsearchCluster::isShared);
         testTask.getOutputs()
             .doNotCacheIf(
                 "Caching disabled for this task since it uses a cluster shared by other tasks",

diff --git a/...ools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java b/...ools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java
@@ -85,7 +85,7 @@ public void apply(Project project) {
             NamedDomainObjectContainer<ElasticsearchCluster> clusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
                 .getExtensions()
                 .getByName(TestClustersPlugin.EXTENSION_NAME);
-            clusters.all(c -> {
+            clusters.configureEach(c -> {
                 if (BuildParams.isInFipsJvm()) {
                     c.setting("xpack.security.transport.ssl.key", "test-node.key");
                     c.keystore("xpack.security.transport.ssl.secure_key_passphrase", "test-node-key-password");

diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy
@@ -166,7 +166,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.output.contains("Task ':myTask' is not up-to-date because:\n" +
-                "  Input property 'clusters.myCluster\$0.nodes.\$0.$propertyName'")
+                "  Input property 'clusters.myCluster\$0.$propertyName'")
         result.output.contains("elasticsearch-keystore script executed!")
         assertEsOutputContains("myCluster", "Starting Elasticsearch process")
         assertEsOutputContains("myCluster", "Stopping node")

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -79,6 +79,7 @@ public void apply(Project project) {
     }
 
     private void setupDistributionContainer(Project project, Property<Boolean> dockerAvailable) {
+
         distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
             Configuration fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name);
             Configuration extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name);

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
@@ -55,7 +55,7 @@ public String toString() {
     private final Property<Boolean> failIfUnavailable;
     private final Property<Boolean> preferArchive;
     private final ConfigurableFileCollection extracted;
-    private Action<ElasticsearchDistribution> distributionFinalizer;
+    private transient Action<ElasticsearchDistribution> distributionFinalizer;
     private boolean frozen = false;
 
     ElasticsearchDistribution(