Merge pull request #71 from Photoroom/ben/1.18_pypi_builds_jeez

[chore] Reverting back to go 1.18 for gopy speed, no more mmap
Photoroom · Jan 22, 2025 · 54e42ce · 54e42ce
2 parents e2072de + a0be396
commit 54e42ce
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 18 deletions.
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -24,7 +24,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.18.1"
 
       - run: go version
 

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.18.1"
 
       - name: Install linux deps
         run: |

diff --git a/go.mod b/go.mod
@@ -1,11 +1,10 @@
 module datago
 
-go 1.22.0
-
-toolchain go1.22.2
+go 1.18
 
 require (
 	github.com/davidbyttow/govips/v2 v2.16.0
+	github.com/go-python/gopy v0.4.10
 	go.uber.org/automaxprocs v1.6.0
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
 )

diff --git a/go.sum b/go.sum
@@ -3,18 +3,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidbyttow/govips/v2 v2.16.0 h1:1nH/Rbx8qZP1hd+oYL9fYQjAnm1+KorX9s07ZGseQmo=
 github.com/davidbyttow/govips/v2 v2.16.0/go.mod h1:clH5/IDVmG5eVyc23qYpyi7kmOT0B/1QNTKtci4RkyM=
+github.com/go-python/gopy v0.4.10 h1:Ec3x+NTSzLsw9f6FTdDLwQCQlmlNmJIu4J6nSnyugqE=
+github.com/go-python/gopy v0.4.10/go.mod h1:zMV/gSSYa9u/8Zp0WYR+L/z+kOIqIUtMg/a1/GRy5uw=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
-github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
 go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
@@ -88,4 +88,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/pkg/worker_filesystem.go b/pkg/worker_filesystem.go
@@ -2,26 +2,33 @@ package datago
 
 import (
 	"fmt"
-
-	"golang.org/x/exp/mmap"
+	"os"
 )
 
 type BackendFileSystem struct {
 	config *DatagoConfig
 }
 
 func loadFromDisk(fsSample fsSampleMetadata, transform *ARAwareTransform, encodeImage bool) *Sample {
-	// Using mmap to put the file directly into memory, removes buffering needs
-	r, err := mmap.Open(fsSample.FilePath)
+	// Load the file into []bytes
+	bytesBuffer, err := os.ReadFile(fsSample.FilePath)
 	if err != nil {
-		panic(err)
+		fmt.Println("Error reading file:", fsSample.FilePath)
+		return nil
 	}
 
-	bytesBuffer := make([]byte, r.Len())
-	_, err = r.ReadAt(bytesBuffer, 0)
-	if err != nil {
-		panic(err)
-	}
+	// Slightly faster take, requires Go 1.21+ which breaks gopy speed for now
+	// // Using mmap to put the file directly into memory, removes buffering needs
+	// r, err := mmap.Open(fsSample.FilePath)
+	// if err != nil {
+	// 	panic(err)
+	// }
+
+	// bytesBuffer := make([]byte, r.Len())
+	// _, err = r.ReadAt(bytesBuffer, 0)
+	// if err != nil {
+	// 	panic(err)
+	// }
 
 	// Decode the image, can error out here also, and return the sample
 	imgPayload, _, err := imageFromBuffer(bytesBuffer, transform, -1., encodeImage, false)