v0.5.0 dev

.github/workflows/go.yml

@@ -0,0 +1,29 @@
+name: 'go' 
+on: # rebuild any PRs and main branch changes
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout (GitHub)
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build devcontainer and run tests
+        uses: devcontainers/[email protected]
+        with:
+          imageName: ghcr.io/${{ github.repository_owner }}/ocfl-go-devcontainer
+          eventFilterForPush: pull_request
+          runCmd: |
+            go test ./... -count=5 -race

examples/listobjects/main.go

@@ -33,65 +33,32 @@ func main() {
 		logger.Error("missing required storage root URI")
 		os.Exit(1)
 	}
-	fsys, dir, err := parseStoreConn(ctx, storeConn)
-	if err != nil {
-		logger.Error("can't parse storage root argument", "err", err)
-		os.Exit(1)
-	}
-	if err := listObjects2(ctx, fsys, dir, numgos, logger); err != nil {
+	if err := listObjects(ctx, storeConn, numgos, logger); err != nil {
 		logger.Error("exit with errors", "err", err)
 		os.Exit(1)
 	}
 }
 
-func listObjects2(ctx context.Context, fsys ocfl.FS, dir string, numgos int, log *slog.Logger) (err error) {
-	allFiles, walkErrFn := ocfl.WalkFiles(ctx, fsys, dir)
-	defer func() {
-		err = walkErrFn()
-	}()
-	decls := allFiles.Filter(func(f *ocfl.FileRef) bool { return f.Namaste().IsObject() })
-	for obj, err := range decls.OpenObjectsBatch(ctx, numgos) {
+func listObjects(ctx context.Context, storeConn string, numgos int, log *slog.Logger) (err error) {
+	fsys, dir, err := parseStoreConn(ctx, storeConn)
+	if err != nil {
+		return fmt.Errorf("can't parse storage root argument: %w", err)
+	}
+	root, err := ocfl.NewRoot(ctx, fsys, dir)
+	if err != nil {
+		return nil
+	}
+	for obj, err := range root.ObjectsBatch(ctx, numgos) {
 		if err != nil {
 			log.Error(err.Error())
 			continue
 		}
 		id := obj.Inventory().ID()
 		fmt.Println(id)
 	}
-	return
+	return nil
 }
 
-// func listObjects(ctx context.Context, fsys ocfl.FS, dir string, gos int, _ *slog.Logger) error {
-// 	objectDirs := func(yield func(string) bool) {
-// 		for dir, err := range ocfl.ObjectPaths(ctx, fsys, dir) {
-// 			if err != nil {
-// 				break
-// 			}
-// 			if !yield(dir) {
-// 				break
-// 			}
-// 		}
-// 	}
-// 	getID := func(dir string) (ocfl.ReadInventory, error) {
-// 		obj, err := ocfl.NewObject(ctx, fsys, dir)
-// 		if err != nil {
-// 			return nil, err
-// 		}
-// 		return obj.Inventory(), nil
-// 	}
-// 	var err error
-// 	resultIter := pipeline.Results(objectDirs, getID, gos)
-// 	resultIter(func(r pipeline.Result[string, ocfl.ReadInventory]) bool {
-// 		if r.Err != nil {
-// 			err = r.Err
-// 			return false
-// 		}
-// 		fmt.Println(r.In, r.Out.ID())
-// 		return true
-// 	})
-// 	return err
-// }
-
 func parseStoreConn(ctx context.Context, name string) (ocfl.FS, string, error) {
 	//if we were using s3-based backend:
 	rl, err := url.Parse(name)

files.go

@@ -230,36 +230,16 @@ type FileErrSeq iter.Seq2[*FileRef, error]
 // UntilErr returns a new iterator yielding *FileRef values from seq that
 // terminates on the first non-nil error in seq. The terminating error is
 // returned by errFn.
-func (seq FileErrSeq) UntilErr() (files FileSeq, errFn func() error) {
-	var firstErr error
-	files = func(yield func(*FileRef) bool) {
-		for file, err := range seq {
-			if err != nil {
-				firstErr = err
-				break
-			}
-			if !yield(file) {
-				break
-			}
-		}
-	}
-	errFn = func() error { return firstErr }
-	return
+func (fileErrs FileErrSeq) UntilErr() (FileSeq, func() error) {
+	files, errFn := seqUntilErr(iter.Seq2[*FileRef, error](fileErrs))
+	return FileSeq(files), errFn
 }
 
 // IgnoreErr returns an iterator of *[FileRef]s in seq that are not associated
 // with an error.
-func (seq FileErrSeq) IgnoreErr() (files FileSeq) {
-	return func(yield func(*FileRef) bool) {
-		for file, err := range seq {
-			if err != nil {
-				continue
-			}
-			if !yield(file) {
-				break
-			}
-		}
-	}
+func (fileErrs FileErrSeq) IgnoreErr() FileSeq {
+	files := seqIgnoreErr(iter.Seq2[*FileRef, error](fileErrs))
+	return FileSeq(files)
 }
 
 // FileDigests is a [FileRef] plus digest values of the file contents.
@@ -386,21 +366,9 @@ func (digests FileDigestsSeq) Stage() (*Stage, error) {
 
 // UntilErr returns an iterator of *FileDigests from dfs that terminates on the
 // first non-nil error in dfs. The terminating error is returned by errFn
-func (dfs FileDigestsErrSeq) UntilErr() (digests FileDigestsSeq, errFn func() error) {
-	var firstErr error
-	digests = func(yield func(*FileDigests) bool) {
-		for file, err := range dfs {
-			if err != nil {
-				firstErr = err
-				break
-			}
-			if !yield(file) {
-				break
-			}
-		}
-	}
-	errFn = func() error { return firstErr }
-	return
+func (dfs FileDigestsErrSeq) UntilErr() (FileDigestsSeq, func() error) {
+	seq, errFn := seqUntilErr(iter.Seq2[*FileDigests, error](dfs))
+	return FileDigestsSeq(seq), errFn
 }
 
 func walkFiles(ctx context.Context, fsys FS, dir string) (FileSeq, func() error) {
@@ -471,3 +439,33 @@ var errBreakFileWalk = errors.New("break")
 func validFileType(mode fs.FileMode) bool {
 	return mode.IsDir() || mode.IsRegular() || mode.Type() == fs.ModeIrregular
 }
+
+func seqUntilErr[T any](inSeq iter.Seq2[T, error]) (outSeq iter.Seq[T], errFn func() error) {
+	var firstErr error
+	outSeq = func(yield func(T) bool) {
+		for v, err := range inSeq {
+			if err != nil {
+				firstErr = err
+				break
+			}
+			if !yield(v) {
+				break
+			}
+		}
+	}
+	errFn = func() error { return firstErr }
+	return
+}
+
+func seqIgnoreErr[T any](inSeq iter.Seq2[T, error]) iter.Seq[T] {
+	return func(yield func(T) bool) {
+		for v, err := range inSeq {
+			if err != nil {
+				continue
+			}
+			if !yield(v) {
+				break
+			}
+		}
+	}
+}