Introduce BlobStore & IdStore APIs

.github/workflows/ci.yaml

@@ -59,6 +59,12 @@ jobs:
       (github.event.pull_request.draft == false)
     runs-on: ubuntu-latest
     steps:
+      - name: Install IPFS Node daemon
+        uses: oduwsdl/setup-ipfs@e92fedca9f61ab9184cb74940254859f4d7af4d9
+        with:
+          ipfs_version: ^0.33
+          run_daemon: true
+
       - uses: actions/checkout@v4
 
       - name: Install bun

MODULE.bazel

@@ -130,6 +130,7 @@ enola_maven.install(
 
         # IPFS & IPLD etc.
         "com.github.ipld:java-cid:1.3.8",
+        "com.github.ipfs:java-ipfs-http-client:d982fc0fa1",
 
         #   NB: Due to https://github.com/bazel-contrib/rules_jvm_external/issues/1324,
         #   this only just so happens to work because this Git SHA starts with a digit.

java/dev/enola/cas/BUILD

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2025 The Enola <https://enola.dev> Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@rules_java//java:defs.bzl", "java_binary")
+load("//tools/bazel:junit.bzl", "junit_tests")
+
+java_binary(
+    name = "cas",
+    srcs = glob(
+        ["*.java"],
+        exclude = [
+            "*Test.java",
+        ],
+    ),
+    visibility = ["//:__subpackages__"],
+    deps = [
+        "//java/dev/enola/common",
+        "//java/dev/enola/common/context",
+        "//java/dev/enola/common/io",
+        "@enola_maven//:com_github_ipfs_java_ipfs_http_client",
+        "@enola_maven//:com_github_ipld_java_cid",
+        "@enola_maven//:com_github_multiformats_java_multibase",
+        "@enola_maven//:com_github_multiformats_java_multihash",
+        "@enola_maven//:com_google_guava_guava",
+        "@enola_maven//:org_jspecify_jspecify",
+        "@enola_maven//:org_slf4j_slf4j_api",
+    ],
+)
+
+junit_tests(
+    name = "tests",
+    size = "medium",
+    srcs = glob(
+        ["*Test.java"],
+        exclude = [],
+    ),
+    srcs_utils = [
+    ],
+    deps = [
+        ":cas",
+        "//java/dev/enola/common/context/testlib",
+        "//java/dev/enola/common/io",
+        "@enola_maven//:com_github_ipfs_java_ipfs_http_client",
+        "@enola_maven//:com_github_ipld_java_cid",
+        "@enola_maven//:org_slf4j_slf4j_jdk14",
+    ],
+)

java/dev/enola/cas/BlobStore.java

@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2025 The Enola <https://enola.dev> Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.enola.cas;
+
+import com.google.common.io.ByteSource;
+
+import io.ipfs.cid.Cid;
+
+import java.io.IOException;
+
+/**
+ * BlobStore stores bytes, which can then be loaded given their CID.
+ *
+ * <p>Note that this interface per-se does not specify anything about how this may be implemented...
+ * just with simple non-distributed local files. Or into a Key Value Store. Or e.g. with <a
+ * href="https://github.com/systemd/casync/">systemd casync</a>. Or may be on <a
+ * href="https://ipfs.tech/>IPFS</a>, either in <a href="https://ipld.io/specs/transport/car/">CAR
+ * files</a>, or it may be distributed (with <a
+ * href="https://docs.ipfs.tech/install/command-line/">IPFS Bitswap with Kubo</a>, or <a
+ * href="https://github.com/Peergos/nabu">Nabu</a>; or perhaps on the <a
+ * href="https://iroh.network/">Iroh Network</a> based on <a
+ * href="https://www.iroh.computer/">Iroh</a> by <a href="https://n0.computer/">number0</a>. Or
+ * something else, maybe based on Consensus with Raft or Paxos.
+ */
+public interface BlobStore {
+
+    Cid store(ByteSource source) throws IOException;
+
+    ByteSource load(Cid cid) throws IOException;
+
+    // TODO void delete(Cid cid)
+}

java/dev/enola/cas/IPFSBlobStore.java

@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2025 The Enola <https://enola.dev> Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.enola.cas;
+
+import com.google.common.io.ByteSource;
+
+import io.ipfs.api.IPFS;
+import io.ipfs.api.NamedStreamable;
+import io.ipfs.cid.Cid;
+
+import java.io.IOException;
+
+public class IPFSBlobStore implements BlobStore { // TODO , IdStore
+
+    private final IPFS ipfs;
+
+    public IPFSBlobStore(IPFS ipfs) {
+        this.ipfs = ipfs;
+    }
+
+    @Override
+    public Cid store(ByteSource source) throws IOException {
+        var namedStreamable = new NamedStreamable.ByteArrayWrapper(source.read());
+        var merkleNode = ipfs.add(namedStreamable);
+        // TODO Why does it not work with version == 1 ?!
+        return Cid.build(0, Cid.Codec.Raw, merkleNode.get(0).hash);
+    }
+
+    @Override
+    public ByteSource load(Cid cid) throws IOException {
+        return ByteSource.wrap(ipfs.cat(cid));
+    }
+}

java/dev/enola/cas/IPFSBlobStoreTest.java

@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2025 The Enola <https://enola.dev> Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.enola.cas;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.io.ByteSource;
+
+import dev.enola.common.context.testlib.SingletonRule;
+import dev.enola.common.io.mediatype.MediaTypeProviders;
+
+import io.ipfs.api.IPFS;
+import io.ipfs.cid.Cid;
+
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Random;
+
+public class IPFSBlobStoreTest {
+
+    public @Rule SingletonRule singleton = SingletonRule.$(MediaTypeProviders.set());
+
+    IPFSBlobStore ipfs = new IPFSBlobStore(new IPFS("/ip4/127.0.0.1/tcp/5001"));
+
+    @Test
+    public void hello() throws IOException {
+        var bytes = ipfs.load(Cid.decode("QmXV7pL1CB7A8Tzk7jP2XE9kRyk8HZd145KDptdxzmNLfu"));
+        assertThat(new String(bytes.read())).isEqualTo("hello, world\n");
+    }
+
+    @Test
+    public void random() throws IOException {
+        var bytes = generateRandomBytes(1024);
+        var cid = ipfs.store(ByteSource.wrap(bytes));
+        var loaded = ipfs.load(cid);
+        assertThat(loaded.read()).isEqualTo(bytes);
+    }
+
+    private static byte[] generateRandomBytes(int length) {
+        byte[] bytes = new byte[length];
+        Random random = new Random();
+        random.nextBytes(bytes);
+        return bytes;
+    }
+}

java/dev/enola/common/io/resource/IPFSResource.java → java/dev/enola/cas/IPFSResource.java

@@ -15,12 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package dev.enola.common.io.resource;
+package dev.enola.cas;
 
 import com.google.common.base.Strings;
 import com.google.common.io.ByteSource;
 import com.google.common.net.MediaType;
 
+import dev.enola.common.io.resource.*;
+
 import io.ipfs.cid.Cid;
 
 import java.net.URI;
@@ -83,7 +85,10 @@ public IPFSResource(
     }
 
     private String ipfs2http(URI ipfsURL, String gateway) {
-        return gateway + ipfsURL.getAuthority() + ipfsURL.getPath();
+        return gateway
+                + (gateway.endsWith("/") ? "" : "/")
+                + ipfsURL.getAuthority()
+                + ipfsURL.getPath();
     }
 
     @Override

java/dev/enola/common/io/resource/IPFSResourceTest.java → java/dev/enola/cas/IPFSResourceTest.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package dev.enola.common.io.resource;
+package dev.enola.cas;
 
 import static com.google.common.truth.Truth.assertThat;
 
@@ -27,6 +27,9 @@
 
 import dev.enola.common.context.testlib.SingletonRule;
 import dev.enola.common.io.mediatype.MediaTypeProviders;
+import dev.enola.common.io.resource.OkHttpResource;
+import dev.enola.common.io.resource.ReadableResource;
+import dev.enola.common.io.resource.ResourceProvider;
 
 import io.ipfs.cid.Cid.CidEncodingException;
 

java/dev/enola/cas/IdStore.java

@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2025 The Enola <https://enola.dev> Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.enola.cas;
+
+import com.google.common.primitives.Bytes;
+
+import dev.enola.common.ByteSeq;
+
+import io.ipfs.cid.Cid;
+
+/** IdStore stores links of IDs of bytes to CIDs. */
+public interface IdStore {
+
+    void link(ByteSeq bytes, Cid cid);
+
+    Cid get(Bytes link);
+}

java/dev/enola/cas/package-info.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2024-2025 The Enola <https://enola.dev> Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * CAS is <a href="https://en.m.wikipedia.org/wiki/Content-addressable_storage">Content-addressable
+ * storage</a>
+ */
+@NullMarked
+package dev.enola.cas;
+
+import org.jspecify.annotations.NullMarked;

java/dev/enola/cli/BUILD

@@ -33,6 +33,7 @@ java_binary(
         "@enola_maven//:com_github_multiformats_java_multihash",
         "//java/dev/enola",
         # TODO Remove most of these direct deps, through new Enola API...
+        "//java/dev/enola/cas",
         "//java/dev/enola/common",
         "//java/dev/enola/common/canonicalize",
         "//java/dev/enola/common/context",

java/dev/enola/cli/CommandWithResourceProvider.java

@@ -20,6 +20,7 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 
+import dev.enola.cas.IPFSResource;
 import dev.enola.common.context.Context;
 import dev.enola.common.io.hashbrown.IntegrityValidatingDelegatingResource;
 import dev.enola.common.io.iri.URIs;

java/dev/enola/common/io/hashbrown/Multihashes.java

@@ -40,7 +40,10 @@ public static HashFunction toGuavaHashFunction(Multihash.Type type) {
         return switch (type) {
             // NB: Please BEWARE of what types are added here; the (sub)selection is intentional!
 
+            // TODO Support blake3, see https://github.com/enola-dev/enola/issues/1125
+
             // TODO Support murmur3-x64-64, https://github.com/multiformats/java-multihash/pull/43
+            //   BUT beware of https://github.com/google/guava/issues/3493, that's worrying!
             //   case murmur3_x64_128 ->  Hashing.murmur3_128();
             // PS: Not suitable, as it's too tiny: case murmur3 -> Hashing.murmur3_32_fixed();
 

java/dev/enola/common/io/resource/ResourceProvider.java

@@ -37,8 +37,8 @@ public interface ResourceProvider extends ProviderFromIRI<Resource> {
 
     // TODO Rename all parameters from iri or uri to url - because that's what these are!
 
-    // TODO Change all @Nullable Resource to Optional<Resource>... or, better, throw exception for
-    // unknown schema
+    // TODO Separate @NonNull SPI provider, instead of changing all @Nullable Resource to
+    // Optional<Resource>... or, better, throw exception for unknown schema
 
     // TODO Should this have a Resource getResource(URI uri, MediaType mediaType) ?