diff --git a/fakestorage/object.go b/fakestorage/object.go index 3ae2cf0cd5..8a251a870c 100644 --- a/fakestorage/object.go +++ b/fakestorage/object.go @@ -6,6 +6,7 @@ package fakestorage import ( "bytes" + "compress/gzip" "encoding/json" "errors" "fmt" @@ -763,8 +764,45 @@ func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) { var content io.Reader content = obj.Content status := http.StatusOK - ranged, start, lastByte, satisfiable := s.handleRange(obj, r) - contentLength := lastByte - start + 1 + + transcoded := false + ranged := false + start := int64(0) + lastByte := int64(0) + satisfiable := true + contentLength := int64(0) + + handledTranscoding := func() bool { + // This should also be false if the Cache-Control metadata field == "no-transform", + // but we don't currently support that field. + // See https://cloud.google.com/storage/docs/transcoding + + if obj.ContentEncoding == "gzip" && !strings.Contains(r.Header.Get("accept-encoding"), "gzip") { + // GCS will transparently decompress gzipped content, see + // https://cloud.google.com/storage/docs/transcoding + // In this case, any Range header is ignored and the full content is returned. + + // If the content is not a valid gzip file, ignore errors and continue + // without transcoding. Otherwise, return decompressed content. + gzipReader, err := gzip.NewReader(content) + if err == nil { + rawContent, err := io.ReadAll(gzipReader) + if err == nil { + transcoded = true + content = bytes.NewReader(rawContent) + contentLength = int64(len(rawContent)) + obj.Size = contentLength + return true + } + } + } + return false + } + + if !handledTranscoding() { + ranged, start, lastByte, satisfiable = s.handleRange(obj, r) + contentLength = lastByte - start + 1 + } if ranged && satisfiable { _, err = obj.Content.Seek(start, io.SeekStart) @@ -793,7 +831,8 @@ func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) { if obj.ContentType != "" { w.Header().Set(contentTypeHeader, obj.ContentType) } - if obj.ContentEncoding != "" { + // If content was transcoded, the underlying encoding was removed so we shouldn't report it. + if obj.ContentEncoding != "" && !transcoded { w.Header().Set("Content-Encoding", obj.ContentEncoding) } } diff --git a/fakestorage/object_test.go b/fakestorage/object_test.go index ac76a7a146..4d3f3b19cc 100644 --- a/fakestorage/object_test.go +++ b/fakestorage/object_test.go @@ -6,6 +6,7 @@ package fakestorage import ( "bytes" + "compress/gzip" "context" "encoding/binary" "errors" @@ -361,6 +362,128 @@ func TestServerClientObjectReader(t *testing.T) { }) } +func TestServerClientObjectTranscoding(t *testing.T) { + const ( + bucketName = "some-bucket" + objectName = "items/data.txt" + content = "some nice content, which will be gziped" + contentType = "text/plain; charset=utf-8" + contentEncoding = "gzip" + ) + + var b bytes.Buffer + gz := gzip.NewWriter(&b) + if _, err := gz.Write([]byte(content)); err != nil { + t.Fatal(err) + } + if err := gz.Flush(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + + objs := []Object{ + { + ObjectAttrs: ObjectAttrs{ + BucketName: bucketName, + Name: objectName, + ContentType: contentType, + ContentEncoding: contentEncoding, + }, + Content: b.Bytes(), + }, + } + + runServersTest(t, runServersOptions{objs: objs}, func(t *testing.T, server *Server) { + client := server.Client() + objHandle := client.Bucket(bucketName).Object(objectName) + reader, err := objHandle.NewReader(context.TODO()) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if string(data) != content { + t.Errorf("wrong data returned\nwant %q\ngot %q", content, string(data)) + } + if ct := reader.Attrs.ContentType; ct != contentType { + t.Errorf("wrong content type\nwant %q\ngot %q", contentType, ct) + } + }) +} + +func TestServerClientObjectSkipTranscoding(t *testing.T) { + const ( + bucketName = "some-bucket" + objectName = "items/data.txt" + content = "some nice content, which will be gziped" + contentType = "text/plain; charset=utf-8" + contentEncoding = "gzip" + ) + + var b bytes.Buffer + gz := gzip.NewWriter(&b) + if _, err := gz.Write([]byte(content)); err != nil { + t.Fatal(err) + } + if err := gz.Flush(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + + objs := []Object{ + { + ObjectAttrs: ObjectAttrs{ + BucketName: bucketName, + Name: objectName, + ContentType: contentType, + ContentEncoding: contentEncoding, + }, + Content: b.Bytes(), + }, + } + + runServersTest(t, runServersOptions{objs: objs}, func(t *testing.T, server *Server) { + client := server.Client() + objHandle := client.Bucket(bucketName).Object(objectName).ReadCompressed(true) // we skip transcoding by `Accept-Encoding: gzip` + reader, err := objHandle.NewReader(context.TODO()) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + // need to unzip manually + gzr, err := gzip.NewReader(reader) + if err != nil { + t.Fatal(err) + } + defer gzr.Close() + + var rawBytes bytes.Buffer + _, err = rawBytes.ReadFrom(gzr) + if err != nil { + t.Fatal(err) + } + + data := rawBytes.Bytes() + + if string(data) != content { + t.Errorf("wrong data returned\nwant %q\ngot %q", content, string(data)) + } + if ct := reader.Attrs.ContentType; ct != contentType { + t.Errorf("wrong content type\nwant %q\ngot %q", contentType, ct) + } + if ct := reader.Attrs.ContentEncoding; ct != contentEncoding { + t.Errorf("wrong content encoding\nwant %q\ngot %q", contentEncoding, ct) + } + }) +} + func TestServerClientObjectRangeReader(t *testing.T) { const ( bucketName = "some-bucket" diff --git a/fakestorage/upload.go b/fakestorage/upload.go index 127ecb2187..d7ded925a7 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -380,10 +380,14 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons if objName == "" { objName = metadata.Name } + if contentEncoding == "" { + contentEncoding = metadata.ContentEncoding + } obj := Object{ ObjectAttrs: ObjectAttrs{ BucketName: bucketName, Name: objName, + ContentType: metadata.ContentType, ContentEncoding: contentEncoding, ACL: getObjectACL(predefinedACL), Metadata: metadata.Metadata, diff --git a/internal/backend/fs.go b/internal/backend/fs.go index 6087fb1914..8a4736b9a3 100644 --- a/internal/backend/fs.go +++ b/internal/backend/fs.go @@ -342,6 +342,7 @@ func (s *storageFS) PatchObject(bucketName, objectName string, metadata map[stri for k, v := range metadata { obj.Metadata[k] = v } + obj.Generation = 0 // reset generation id return s.CreateObject(obj) // recreate object } @@ -356,7 +357,7 @@ func (s *storageFS) UpdateObject(bucketName, objectName string, metadata map[str for k, v := range metadata { obj.Metadata[k] = v } - obj.Generation = 0 + obj.Generation = 0 // reset generation id return s.CreateObject(obj) // recreate object }