From ed9f368c5d2aa93348c35a6aed08a18bb827a427 Mon Sep 17 00:00:00 2001 From: Nick Hale <4175918+njhale@users.noreply.github.com> Date: Wed, 12 Feb 2025 17:15:52 -0500 Subject: [PATCH] enhance: use workspace file revisions to avoid clobbering the database Signed-off-by: Nick Hale <4175918+njhale@users.noreply.github.com> --- database/go.mod | 16 +++++++----- database/go.sum | 33 ++++++++++++----------- database/main.go | 58 ++++++++++++++++++++++++++++++++++------- workspace-files/go.mod | 2 ++ workspace-files/go.sum | 8 +++--- workspace-files/main.go | 12 ++++----- 6 files changed, 86 insertions(+), 43 deletions(-) diff --git a/database/go.mod b/database/go.mod index 623febe5..c3bdd88e 100644 --- a/database/go.mod +++ b/database/go.mod @@ -2,21 +2,23 @@ module obot-platform/database go 1.23.3 +replace github.com/gptscript-ai/go-gptscript => github.com/thedadams/go-gptscript v0.0.0-20250219113618-25d959a071ff + require ( - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/gptscript-ai/go-gptscript v0.9.6-0.20250204133419-744b25b84a61 + github.com/gptscript-ai/go-gptscript v0.9.6-0.20250211184209-834896a4bb9f github.com/ncruces/go-sqlite3 v0.20.3 ) require ( - github.com/getkin/kin-openapi v0.124.0 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/swag v0.22.8 // indirect - github.com/invopop/yaml v0.2.0 // indirect + github.com/getkin/kin-openapi v0.129.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/ncruces/julianday v1.0.0 // indirect + github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 // indirect + github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/tetratelabs/wazero v1.8.2 // indirect golang.org/x/sys v0.27.0 // indirect diff --git a/database/go.sum b/database/go.sum index 15473198..736f4c0b 100644 --- a/database/go.sum +++ b/database/go.sum @@ -1,43 +1,43 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M= -github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= -github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= +github.com/getkin/kin-openapi v0.129.0 h1:QGYTNcmyP5X0AtFQ2Dkou9DGBJsUETeLH9rFrJXZh30= +github.com/getkin/kin-openapi v0.129.0/go.mod h1:gmWI+b/J45xqpyK5wJmRRZse5wefA5H0RDMK46kLUtI= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/gptscript-ai/go-gptscript v0.9.6-0.20250204133419-744b25b84a61 h1:QxLjsLOYlsVLPwuRkP0Q8EcAoZT1s8vU2ZBSX0+R6CI= -github.com/gptscript-ai/go-gptscript v0.9.6-0.20250204133419-744b25b84a61/go.mod h1:/FVuLwhz+sIfsWUgUHWKi32qT0i6+IXlUlzs70KKt/Q= -github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= -github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/ncruces/go-sqlite3 v0.20.3 h1:+4G4uEqOeusF0yRuQVUl9fuoEebUolwQSnBUjYBLYIw= github.com/ncruces/go-sqlite3 v0.20.3/go.mod h1:ojLIAB243gtz68Eo283Ps+k9PyR3dvzS+9/RgId4+AA= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= +github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 h1:nZspmSkneBbtxU9TopEAE0CY+SBJLxO8LPUlw2vG4pU= +github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80/go.mod h1:7tFDb+Y51LcDpn26GccuUgQXUk6t0CXZsivKjyimYX8= +github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 h1:t05Ww3DxZutOqbMN+7OIuqDwXbhl32HiZGpLy26BAPc= +github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4= github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= +github.com/thedadams/go-gptscript v0.0.0-20250219113618-25d959a071ff h1:mywQbNqkwJEWtNOF6YhUQnqRlccwX7Ipe5tJ6cKZ5bs= +github.com/thedadams/go-gptscript v0.0.0-20250219113618-25d959a071ff/go.mod h1:QvGPZoRuAiA8P5EzPI05kTrs+LZ0ipHywUGsKruSknw= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= @@ -47,6 +47,5 @@ golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/database/main.go b/database/main.go index 1700a11b..614218aa 100644 --- a/database/main.go +++ b/database/main.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "os" + "slices" "obot-platform/database/pkg/cmd" @@ -24,7 +25,11 @@ func main() { } command := os.Args[1] - g, err := gptscript.NewGPTScript() + g, err := gptscript.NewGPTScript( + gptscript.GlobalOptions{ + WorkspaceTool: "github.com/thedadams/workspace-provider@open-file-with-revision-id", + }, + ) if err != nil { fmt.Printf("Error creating GPTScript: %v\n", err) os.Exit(1) @@ -32,15 +37,17 @@ func main() { defer g.Close() var ( - ctx = context.Background() - dbFileName = "acorn.db" - dbWorkspacePath = "/databases/" + dbFileName + ctx = context.Background() + dbFileName = "acorn.db" + dbWorkspacePath = "/databases/" + dbFileName + revisionID string = "-1" + initialDBData []byte ) - // Read the database file from the workspace - initialDBData, err := g.ReadFileInWorkspace(ctx, dbWorkspacePath, gptscript.ReadFileInWorkspaceOptions{ + workspaceDB, err := g.ReadFileWithRevisionInWorkspace(ctx, dbWorkspacePath, gptscript.ReadFileInWorkspaceOptions{ WorkspaceID: workspaceID, }) + var notFoundErr *gptscript.NotFoundInWorkspaceError if err != nil && !errors.As(err, ¬FoundErr) { fmt.Printf("Error reading DB file: %v\n", err) @@ -57,11 +64,15 @@ func main() { defer os.Remove(dbFile.Name()) // Write the data to the temporary file - if initialDBData != nil { + if workspaceDB != nil && workspaceDB.Content != nil { + initialDBData = workspaceDB.Content if err := os.WriteFile(dbFile.Name(), initialDBData, 0644); err != nil { fmt.Printf("Error writing to temp file: %v\n", err) os.Exit(1) } + if workspaceDB.RevisionID != "" { + revisionID = workspaceDB.RevisionID + } } // Run the requested command @@ -74,7 +85,7 @@ func main() { case "runDatabaseSQL": result, err = cmd.RunDatabaseCommand(ctx, dbFile, os.Getenv("SQL"), "-header") if err == nil { - err = saveWorkspaceDB(ctx, g, dbWorkspacePath, dbFile, initialDBData) + err = saveWorkspaceDB(ctx, g, dbWorkspacePath, revisionID, dbFile, initialDBData) } case "databaseContext": result, err = cmd.DatabaseContext(ctx, dbFile) @@ -95,6 +106,7 @@ func saveWorkspaceDB( ctx context.Context, g *gptscript.GPTScript, dbWorkspacePath string, + revisionID string, dbFile *os.File, initialDBData []byte, ) error { @@ -108,11 +120,39 @@ func saveWorkspaceDB( } if err := g.WriteFileInWorkspace(ctx, dbWorkspacePath, updatedDBData, gptscript.WriteFileInWorkspaceOptions{ - WorkspaceID: workspaceID, + WorkspaceID: workspaceID, + CreateRevision: &([]bool{true}[0]), + LatestRevisionID: revisionID, }); err != nil { return fmt.Errorf("Error writing updated DB file to workspace: %v", err) } + // Delete old revisions after successfully writing the new revision + revisions, err := g.ListRevisionsForFileInWorkspace(ctx, dbWorkspacePath, gptscript.ListRevisionsForFileInWorkspaceOptions{ + WorkspaceID: workspaceID, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Error listing revisions: %v\n", err) + return nil + } + + // Delete all revisions older than the one we just used as LatestRevisionID + lastRevisionIndex := slices.IndexFunc(revisions, func(rev gptscript.FileInfo) bool { + return rev.RevisionID == revisionID + }) + + if lastRevisionIndex < 0 { + return nil + } + + for _, rev := range revisions[:lastRevisionIndex+1] { + if err := g.DeleteRevisionForFileInWorkspace(ctx, dbWorkspacePath, rev.RevisionID, gptscript.DeleteRevisionForFileInWorkspaceOptions{ + WorkspaceID: workspaceID, + }); err != nil { + fmt.Fprintf(os.Stderr, "Error deleting revision %s: %v\n", rev.RevisionID, err) + } + } + return nil } diff --git a/workspace-files/go.mod b/workspace-files/go.mod index d561a0eb..4af7eef7 100644 --- a/workspace-files/go.mod +++ b/workspace-files/go.mod @@ -2,6 +2,8 @@ module github.com/obot-platform/tools/context go 1.23.2 +replace github.com/gptscript-ai/go-gptscript => github.com/thedadams/go-gptscript v0.0.0-20250213160028-965e8afaa113 + require github.com/gptscript-ai/go-gptscript v0.9.6-0.20250204133419-744b25b84a61 require ( diff --git a/workspace-files/go.sum b/workspace-files/go.sum index f6e3b810..029e6487 100644 --- a/workspace-files/go.sum +++ b/workspace-files/go.sum @@ -8,8 +8,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gptscript-ai/go-gptscript v0.9.6-0.20250204133419-744b25b84a61 h1:QxLjsLOYlsVLPwuRkP0Q8EcAoZT1s8vU2ZBSX0+R6CI= -github.com/gptscript-ai/go-gptscript v0.9.6-0.20250204133419-744b25b84a61/go.mod h1:/FVuLwhz+sIfsWUgUHWKi32qT0i6+IXlUlzs70KKt/Q= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -30,8 +28,10 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/thedadams/go-gptscript v0.0.0-20250213160028-965e8afaa113 h1:YH/7QBzjrI4LPyNJG4FQPd5nWBBAHqkFC5WcCsKeUsk= +github.com/thedadams/go-gptscript v0.0.0-20250213160028-965e8afaa113/go.mod h1:QvGPZoRuAiA8P5EzPI05kTrs+LZ0ipHywUGsKruSknw= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/workspace-files/main.go b/workspace-files/main.go index e1888c94..25ec0582 100644 --- a/workspace-files/main.go +++ b/workspace-files/main.go @@ -174,17 +174,17 @@ func read(ctx context.Context, filename string) error { return err } - data, err := client.ReadFileInWorkspace(ctx, path.Join(FilesDir, filename)) + response, err := client.ReadFileInWorkspace(ctx, path.Join(FilesDir, filename)) if err != nil { return err } - if len(data) > MaxFileSize { + if len(response.Content) > MaxFileSize { return fmt.Errorf("file size exceeds %d bytes", MaxFileSize) } - if utf8.Valid(data) { - fmt.Println(string(data)) + if utf8.Valid(response.Content) { + fmt.Println(string(response.Content)) return nil } @@ -206,10 +206,10 @@ func copy(ctx context.Context, filename, toFilename string) error { return err } - data, err := client.ReadFileInWorkspace(ctx, path.Join(FilesDir, filename)) + response, err := client.ReadFileInWorkspace(ctx, path.Join(FilesDir, filename)) if err != nil { return err } - return client.WriteFileInWorkspace(ctx, path.Join(FilesDir, toFilename), data) + return client.WriteFileInWorkspace(ctx, path.Join(FilesDir, toFilename), response.Content) }