feat: optimize resolving manifests (#293)

* provide possibility to resolve HTTP and file manifests

* declare task output

* checkstyle

plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/AbstractManifestResolveTask.java

@@ -18,6 +18,7 @@
 import org.gradle.api.DefaultTask;
 import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.tasks.Internal;
+import org.gradle.api.tasks.OutputDirectory;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.options.Option;
 
@@ -64,7 +65,7 @@ public void resolveAutodocManifest() {
         getProject().getConfigurations()
                 .stream().flatMap(config -> config.getDependencies().stream())
                 .distinct()
-                .filter(this::dependencyFilter)
+                .filter(this::includeDependency)
                 .filter(dep -> !getExclusions().contains(dep.getName()))
                 .map(this::createSource)
                 .filter(Optional::isPresent)
@@ -76,7 +77,20 @@ public void setOutput(String output) {
         this.outputDirectoryOverride = new File(output);
     }
 
-    protected abstract boolean dependencyFilter(Dependency dependency);
+    @OutputDirectory
+    public File getOutputFile() {
+        return downloadDirectory.toFile();
+    }
+
+    /**
+     * Whether to consider a particular dependency for manifest resolution.
+     *
+     * @param dependency The dependency in question
+     * @return true if it should be considered, false otherwise.
+     */
+    protected boolean includeDependency(Dependency dependency) {
+        return true;
+    }
 
     /**
      * Returns an {@link InputStream} that points to the physical location of the autodoc manifest file.

plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DependencySource.java

@@ -16,28 +16,70 @@
 
 import org.gradle.api.artifacts.Dependency;
 
+import java.io.InputStream;
 import java.net.URI;
 
 import static java.lang.String.format;
 
 /**
  * Represents the combination of a dependency and a pointer (URL) to its physical location.
- *
- * @param dependency the dependency in question
- * @param uri        the location where the physical file exists
- * @param classifier what type of dependency we have, e.g. sources, sources, manifest etc
- * @param type       file extension
  */
-public record DependencySource(Dependency dependency, URI uri, String classifier, String type) {
-    @Override
-    public String toString() {
-        return "{" +
-                "dependency=" + dependency +
-                ", uri=" + uri +
-                '}';
+public abstract class DependencySource {
+    private final Dependency dependency;
+    private final URI uri;
+    private final String classifier;
+    private final String type;
+
+    /**
+     * base constructor for a {@link DependencySource}
+     *
+     * @param dependency the dependency in question
+     * @param uri        the location where the physical file exists
+     * @param classifier what type of dependency we have, e.g. sources, sources, manifest etc
+     * @param type       file extension
+     */
+    public DependencySource(Dependency dependency, URI uri, String classifier, String type) {
+        this.dependency = dependency;
+        this.uri = uri;
+        this.classifier = classifier;
+        this.type = type;
     }
 
+    /**
+     * constructs the filename NAME-VERSION-CLASSIFIER.TYPE
+     */
     String filename() {
         return format("%s-%s-%s.%s", dependency.getName(), dependency.getVersion(), classifier, type);
     }
+
+    /**
+     * Checks whether a dependency exists. In some implementations this may involve remote calls, so use this with prejudice!
+     *
+     * @return whether the dependency exists, i.e. the {@link DependencySource#uri()} points to a valid file
+     */
+    public abstract boolean exists();
+
+    public Dependency dependency() {
+        return dependency;
+    }
+
+    public URI uri() {
+        return uri;
+    }
+
+    public String classifier() {
+        return classifier;
+    }
+
+    public String type() {
+        return type;
+    }
+
+    /**
+     * Opens an input stream to the file located at {@link DependencySource#uri()}. It is highly recommended to check {@link DependencySource#exists()}
+     * beforehand.
+     *
+     * @return Either the input stream to the file, or {@code null} if failed.
+     */
+    public abstract InputStream inputStream();
 }

plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DependencySourceFactory.java

@@ -0,0 +1,31 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.plugins.autodoc.tasks;
+
+import org.gradle.api.artifacts.Dependency;
+
+import java.net.URI;
+
+class DependencySourceFactory {
+    public static DependencySource createDependencySource(URI uri, Dependency dependency, String classifier, String type) {
+        if (uri.getScheme().equals("file")) {
+            return new FileSource(dependency, uri, classifier, type);
+        } else if (uri.getScheme().startsWith("http")) {
+            return new HttpSource(dependency, uri, classifier, type);
+        } else {
+            throw new RuntimeException("Unknown URI scheme " + uri);
+        }
+    }
+}

plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DownloadManifestTask.java

@@ -15,15 +15,12 @@
 package org.eclipse.edc.plugins.autodoc.tasks;
 
 import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.ProjectDependency;
 import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
-import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
 import java.nio.file.Files;
 import java.time.Duration;
 import java.time.Instant;
@@ -37,31 +34,19 @@ public class DownloadManifestTask extends AbstractManifestResolveTask {
     public static final String NAME = "downloadManifests";
     private static final Duration MAX_MANIFEST_AGE = Duration.ofHours(24);
 
-    private final HttpClient httpClient;
-
-    public DownloadManifestTask() {
-        httpClient = HttpClient.newHttpClient();
-    }
-
     @Override
-    protected boolean dependencyFilter(Dependency dependency) {
-        return !(dependency instanceof DefaultProjectDependency);
+    protected boolean includeDependency(Dependency dependency) {
+        return !(dependency instanceof ProjectDependency);
     }
 
     @Override
     protected InputStream resolveManifest(DependencySource autodocManifest) {
-        var request = HttpRequest.newBuilder().uri(autodocManifest.uri()).GET().build();
-        HttpResponse<InputStream> response;
-        try {
-            response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
-        } catch (IOException | InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-        if (response.statusCode() != 200) {
-            getLogger().warn("Could not download {}, HTTP response: {}", autodocManifest.dependency(), response);
+        var inputStream = autodocManifest.inputStream();
+        if (inputStream == null) {
+            getLogger().warn("Could not obtain {}", autodocManifest.dependency());
             return null;
         }
-        return response.body();
+        return inputStream;
     }
 
     /**
@@ -79,7 +64,7 @@ protected InputStream resolveManifest(DependencySource autodocManifest) {
     @Override
     protected Optional<DependencySource> createSource(Dependency dependency) {
         if (isLocalFileValid(dependency)) {
-            getLogger().debug("Local file {} was deemed to be viable, will not download", new DependencySource(dependency, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename());
+            getLogger().debug("Local file {} was deemed to be viable, will not download", dependency);
             return Optional.empty();
         }
         var repos = getProject().getRepositories().stream().toList();
@@ -89,18 +74,14 @@ protected Optional<DependencySource> createSource(Dependency dependency) {
                 .map(repo -> {
                     var repoUrl = createArtifactUrl(dependency, repo);
                     try {
-                        // we use a HEAD request, because we only want to see whether that module has a `-manifest.json`
-                        var uri = URI.create(repoUrl);
-                        var headRequest = HttpRequest.newBuilder()
-                                .uri(uri)
-                                .method("HEAD", HttpRequest.BodyPublishers.noBody())
-                                .build();
-                        var response = httpClient.send(headRequest, HttpResponse.BodyHandlers.discarding());
-                        if (response.statusCode() == 200) {
-                            return new DependencySource(dependency, uri, MANIFEST_CLASSIFIER, MANIFEST_TYPE);
+                        var ds = DependencySourceFactory.createDependencySource(URI.create(repoUrl), dependency, MANIFEST_CLASSIFIER, MANIFEST_TYPE);
+                        if (ds.exists()) {
+                            getLogger().debug("Manifest found for '{}' at {}", dependency.getName(), ds.uri());
+                            return ds;
                         }
+                        getLogger().debug("Manifest not found for '{}' at {}", dependency.getName(), ds.uri());
                         return null;
-                    } catch (IOException | InterruptedException | IllegalArgumentException e) {
+                    } catch (IllegalArgumentException e) {
                         return null;
                     }
                 })
@@ -126,7 +107,8 @@ private String createArtifactUrl(Dependency dep, MavenArtifactRepository repo) {
      */
     private boolean isLocalFileValid(Dependency dep) {
         if (!downloadDirectory.toFile().exists()) return false;
-        var filePath = downloadDirectory.resolve(new DependencySource(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename());
+        var filename = format("%s-%s-%s.%s", dep.getName(), dep.getVersion(), MANIFEST_CLASSIFIER, MANIFEST_TYPE);
+        var filePath = downloadDirectory.resolve(filename);
         var file = filePath.toFile();
         if (!file.exists() || !file.canRead()) return false;
 
@@ -137,6 +119,4 @@ private boolean isLocalFileValid(Dependency dep) {
             throw new RuntimeException(e);
         }
     }
-
-
 }

plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/FileSource.java

@@ -0,0 +1,56 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.plugins.autodoc.tasks;
+
+import org.gradle.api.artifacts.Dependency;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * A dependency that is represented in the local file system, e.g. the local Maven cache
+ */
+public class FileSource extends DependencySource {
+    /**
+     * Instantiates a new file source
+     *
+     * @param dependency the dependency in question
+     * @param uri        the location where the physical file exists
+     * @param classifier what type of dependency we have, e.g. sources, sources, manifest etc
+     * @param type       file extension
+     */
+    public FileSource(Dependency dependency, URI uri, String classifier, String type) {
+        super(dependency, uri, classifier, type);
+    }
+
+    @Override
+    public boolean exists() {
+        return Files.exists(Path.of(uri()));
+    }
+
+    @Override
+    public InputStream inputStream() {
+        try {
+            return new FileInputStream(new File(uri()));
+        } catch (FileNotFoundException e) {
+            return null;
+        }
+    }
+}