From a0be3966f962b483a367e684de1f7180beb4ad87 Mon Sep 17 00:00:00 2001 From: Benjamin Lefaudeux Date: Wed, 22 Jan 2025 11:32:57 +0000 Subject: [PATCH] reverting back to go 1.18 for gopy speed, no more mmap --- .github/workflows/ci-cd.yml | 2 +- .github/workflows/go.yml | 2 +- go.mod | 5 ++--- go.sum | 5 ++--- pkg/worker_filesystem.go | 27 +++++++++++++++++---------- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index cceae3f..cd0951b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.22" + go-version: "1.18.1" - run: go version diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9d147a1..b7919f9 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.22" + go-version: "1.18.1" - name: Install linux deps run: | diff --git a/go.mod b/go.mod index 5543689..dd1438d 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,10 @@ module datago -go 1.22.0 - -toolchain go1.22.2 +go 1.18 require ( github.com/davidbyttow/govips/v2 v2.16.0 + github.com/go-python/gopy v0.4.10 go.uber.org/automaxprocs v1.6.0 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 ) diff --git a/go.sum b/go.sum index c9b19d4..0869e18 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidbyttow/govips/v2 v2.16.0 h1:1nH/Rbx8qZP1hd+oYL9fYQjAnm1+KorX9s07ZGseQmo= github.com/davidbyttow/govips/v2 v2.16.0/go.mod h1:clH5/IDVmG5eVyc23qYpyi7kmOT0B/1QNTKtci4RkyM= +github.com/go-python/gopy v0.4.10 h1:Ec3x+NTSzLsw9f6FTdDLwQCQlmlNmJIu4J6nSnyugqE= +github.com/go-python/gopy v0.4.10/go.mod h1:zMV/gSSYa9u/8Zp0WYR+L/z+kOIqIUtMg/a1/GRy5uw= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -10,11 +12,9 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= @@ -88,4 +88,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/worker_filesystem.go b/pkg/worker_filesystem.go index 1082618..5063f5d 100644 --- a/pkg/worker_filesystem.go +++ b/pkg/worker_filesystem.go @@ -2,8 +2,7 @@ package datago import ( "fmt" - - "golang.org/x/exp/mmap" + "os" ) type BackendFileSystem struct { @@ -11,17 +10,25 @@ type BackendFileSystem struct { } func loadFromDisk(fsSample fsSampleMetadata, transform *ARAwareTransform, encodeImage bool) *Sample { - // Using mmap to put the file directly into memory, removes buffering needs - r, err := mmap.Open(fsSample.FilePath) + // Load the file into []bytes + bytesBuffer, err := os.ReadFile(fsSample.FilePath) if err != nil { - panic(err) + fmt.Println("Error reading file:", fsSample.FilePath) + return nil } - bytesBuffer := make([]byte, r.Len()) - _, err = r.ReadAt(bytesBuffer, 0) - if err != nil { - panic(err) - } + // Slightly faster take, requires Go 1.21+ which breaks gopy speed for now + // // Using mmap to put the file directly into memory, removes buffering needs + // r, err := mmap.Open(fsSample.FilePath) + // if err != nil { + // panic(err) + // } + + // bytesBuffer := make([]byte, r.Len()) + // _, err = r.ReadAt(bytesBuffer, 0) + // if err != nil { + // panic(err) + // } // Decode the image, can error out here also, and return the sample imgPayload, _, err := imageFromBuffer(bytesBuffer, transform, -1., encodeImage, false)