From 6c1b3119ace88421cc97416ead789ecd55e6b172 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 11 Apr 2024 14:51:31 -0500 Subject: [PATCH 01/18] Update test data to match recent changes --- testdb.sql | 78 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/testdb.sql b/testdb.sql index 892aa83..d823ba6 100644 --- a/testdb.sql +++ b/testdb.sql @@ -14,6 +14,7 @@ DROP TABLE IF EXISTS msgs_broadcast CASCADE; DROP TABLE IF EXISTS msgs_label CASCADE; DROP TABLE IF EXISTS msgs_msg_labels CASCADE; DROP TABLE IF EXISTS msgs_msg CASCADE; +DROP TABLE IF EXISTS msgs_optin CASCADE; DROP TABLE IF EXISTS ivr_call CASCADE; DROP TABLE IF EXISTS contacts_contacturn CASCADE; DROP TABLE IF EXISTS contacts_contactgroup_contacts CASCADE; @@ -30,6 +31,11 @@ CREATE TABLE orgs_org ( created_on timestamp with time zone NOT NULL ); +CREATE TABLE auth_user ( + id serial primary key, + username character varying(128) NOT NULL +); + CREATE TABLE channels_channel ( id serial primary key, uuid character varying(36) NOT NULL, @@ -84,33 +90,6 @@ CREATE TABLE flows_flow ( name character varying(128) NOT NULL ); -CREATE TABLE msgs_msg ( - id serial primary key, - uuid uuid NULL, - org_id integer NOT NULL REFERENCES orgs_org(id), - broadcast_id integer NULL, - text text NOT NULL, - high_priority boolean NULL, - created_on timestamp with time zone NOT NULL, - modified_on timestamp with time zone NOT NULL, - sent_on timestamp with time zone, - queued_on timestamp with time zone, - direction character varying(1) NOT NULL, - status character varying(1) NOT NULL, - visibility character varying(1) NOT NULL, - msg_type character varying(1), - msg_count integer NOT NULL, - error_count integer NOT NULL, - next_attempt timestamp with time zone NOT NULL, - external_id character varying(255), - attachments character varying(255)[], - channel_id integer REFERENCES channels_channel(id), - contact_id integer NOT NULL REFERENCES contacts_contact(id), - contact_urn_id integer NULL REFERENCES contacts_contacturn(id), - flow_id integer NULL REFERENCES flows_flow(id), - metadata text -); - CREATE TABLE msgs_broadcast ( id serial primary key, org_id integer NOT NULL REFERENCES orgs_org(id), @@ -138,6 +117,46 @@ CREATE TABLE msgs_broadcastmsgcount ( broadcast_id integer NOT NULL REFERENCES msgs_broadcast(id) ); +CREATE TABLE msgs_optin ( + id serial PRIMARY KEY, + uuid uuid NOT NULL, + org_id integer NOT NULL REFERENCES orgs_org(id) ON DELETE CASCADE, + name character varying(64) +); + +CREATE TABLE msgs_msg ( + id bigserial PRIMARY KEY, + uuid uuid NOT NULL, + org_id integer NOT NULL REFERENCES orgs_org(id) ON DELETE CASCADE, + channel_id integer REFERENCES channels_channel(id) ON DELETE CASCADE, + contact_id integer NOT NULL REFERENCES contacts_contact(id) ON DELETE CASCADE, + contact_urn_id integer REFERENCES contacts_contacturn(id) ON DELETE CASCADE, + broadcast_id integer REFERENCES msgs_broadcast(id) ON DELETE CASCADE, + flow_id integer REFERENCES flows_flow(id) ON DELETE CASCADE, + --ticket_id integer REFERENCES tickets_ticket(id) ON DELETE CASCADE, + created_by_id integer REFERENCES auth_user(id) ON DELETE CASCADE, + text text NOT NULL, + attachments character varying(255)[] NULL, + quick_replies character varying(64)[] NULL, + optin_id integer REFERENCES msgs_optin(id) ON DELETE CASCADE, + locale character varying(6) NULL, + created_on timestamp with time zone NOT NULL, + modified_on timestamp with time zone NOT NULL, + sent_on timestamp with time zone, + msg_type character varying(1) NOT NULL, + direction character varying(1) NOT NULL, + status character varying(1) NOT NULL, + visibility character varying(1) NOT NULL, + msg_count integer NOT NULL, + high_priority boolean NULL, + error_count integer NOT NULL, + next_attempt timestamp with time zone NOT NULL, + failed_reason character varying(1), + external_id character varying(255), + metadata text, + log_uuids uuid[] +); + CREATE TABLE msgs_label ( id serial primary key, uuid character varying(36) NULL, @@ -150,11 +169,6 @@ CREATE TABLE msgs_msg_labels ( label_id integer NOT NULL REFERENCES msgs_label(id) ); -CREATE TABLE auth_user ( - id serial primary key, - username character varying(128) NOT NULL -); - CREATE TABLE ivr_call ( id serial primary key, org_id integer NOT NULL REFERENCES orgs_org(id), From 3922b3d1d325586e86c4a7e19aa7efdca6d64722 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 11 Apr 2024 15:06:47 -0500 Subject: [PATCH 02/18] Update to go 1.22 and update deps --- .github/workflows/ci.yml | 2 +- go.mod | 18 +++++++++--------- go.sum | 39 ++++++++++++++++++++++----------------- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad1418..62da6ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: "1.21.x" + go-version: "1.22.x" jobs: test: name: Test diff --git a/go.mod b/go.mod index dc2b97d..fcc34de 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module github.com/nyaruka/rp-archiver -go 1.21 +go 1.22 require ( - github.com/aws/aws-sdk-go v1.49.15 + github.com/aws/aws-sdk-go v1.51.20 github.com/evalphobia/logrus_sentry v0.8.2 github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.9 - github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.42.7 + github.com/nyaruka/ezconf v0.3.0 + github.com/nyaruka/gocommon v1.53.2 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 ) require ( @@ -25,11 +25,11 @@ require ( github.com/naoina/toml v0.1.1 // indirect github.com/nyaruka/librato v1.1.1 // indirect github.com/nyaruka/null/v2 v2.0.3 // indirect - github.com/nyaruka/phonenumbers v1.3.0 // indirect + github.com/nyaruka/phonenumbers v1.3.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 7b83166..2a84d2f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k= -github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.51.20 h1:ziM90ujYHKKkoTZL+Wg2LwjbQecL+l298GGJeG4ktZs= +github.com/aws/aws-sdk-go v1.51.20/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -7,7 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFetyN46VoDQ= github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc= -github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= @@ -33,16 +32,16 @@ github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hz github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= -github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.42.7 h1:4U7Ta1LIHVc/uv8sfqmmV5oRiFU8TcJM9a7QjxVoaeA= -github.com/nyaruka/gocommon v1.42.7/go.mod h1:DMj0TJPT2zi6eoXrBSsJTGBxSAUkpBk+UzcMyAbq5DA= +github.com/nyaruka/ezconf v0.3.0 h1:kGvJqVN8AHowb4HdaHAviJ0Z3yI5Pyekp1WqibFEaGk= +github.com/nyaruka/ezconf v0.3.0/go.mod h1:89GUW6EPRNLIxT7lC4LWnjWTgZeQwRoX7lBmc8ralAU= +github.com/nyaruka/gocommon v1.53.2 h1:DKXlvNUcim/+4X0Dz8BjdDHmQBNVU60giEOVu2faRCE= +github.com/nyaruka/gocommon v1.53.2/go.mod h1:1E8KKcJ4r+FPWQm5ImGjiH6K96dKInjLM9BkcD4zOUQ= github.com/nyaruka/librato v1.1.1 h1:0nTYtJLl3Sn7lX3CuHsLf+nXy1k/tGV0OjVxLy3Et4s= github.com/nyaruka/librato v1.1.1/go.mod h1:fme1Fu1PT2qvkaBZyw8WW+SrnFe2qeeCWpvqmAaKAKE= github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw= github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA= -github.com/nyaruka/phonenumbers v1.3.0 h1:IFyyJfF2Elg8xGKFghWrRXzb6qAHk+Q3uPqmIgS20JQ= -github.com/nyaruka/phonenumbers v1.3.0/go.mod h1:4jyKp/BFUokLbCHyoZag+T3S1KezFVoEKtgnbpzItC4= +github.com/nyaruka/phonenumbers v1.3.4 h1:bF1Wdh++fxw09s3surhVeBhXEcUKG07pHeP8HQXqjn8= +github.com/nyaruka/phonenumbers v1.3.4/go.mod h1:Ut+eFwikULbmCenH6InMKL9csUNLyxHuBLyfkpum11s= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -53,17 +52,23 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= From a317dfd3f608c392c31d1a037be2a958add133a9 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 13 Oct 2023 13:47:16 +0200 Subject: [PATCH 03/18] Start replacing logrus with slog --- archives/archives.go | 127 +++++++++++++++++++------------------- archives/archives_test.go | 5 +- archives/messages.go | 31 +++++----- archives/runs.go | 28 ++++----- archives/s3.go | 6 +- cmd/rp-archiver/main.go | 28 ++++++--- utils/logrus.go | 92 +++++++++++++++++++++++++++ 7 files changed, 208 insertions(+), 109 deletions(-) create mode 100644 utils/logrus.go diff --git a/archives/archives.go b/archives/archives.go index 76f8dbc..a7c65f9 100644 --- a/archives/archives.go +++ b/archives/archives.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "io" + "log/slog" "os" "path/filepath" "time" @@ -19,7 +20,6 @@ import ( "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dates" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) // ArchiveType is the type for the archives @@ -471,13 +471,13 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi start := dates.Now() - log := logrus.WithFields(logrus.Fields{ - "org_id": archive.Org.ID, - "archive_type": archive.ArchiveType, - "start_date": archive.StartDate, - "end_date": archive.endDate(), - "period": archive.Period, - }) + log := slog.With( + "org_id", archive.Org.ID, + "archive_type", archive.ArchiveType, + "start_date", archive.StartDate, + "end_date", archive.endDate(), + "period", archive.Period, + ) filename := fmt.Sprintf("%s_%d_%s%d%02d%02d_", archive.ArchiveType, archive.Org.ID, archive.Period, archive.StartDate.Year(), archive.StartDate.Month(), archive.StartDate.Day()) file, err := os.CreateTemp(archivePath, filename) @@ -490,7 +490,7 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi if archive.ArchiveFile == "" { err = os.Remove(file.Name()) if err != nil { - log.WithError(err).WithField("filename", file.Name()).Error("error cleaning up archive file") + log.Error("error cleaning up archive file", "error", err, "filename", file.Name()) } } }() @@ -500,9 +500,7 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi writer := bufio.NewWriter(gzWriter) defer file.Close() - log.WithFields(logrus.Fields{ - "filename": file.Name(), - }).Debug("creating new archive file") + log.Debug("creating new archive file", "filename", file.Name()) recordCount := 0 switch archive.ArchiveType { @@ -540,13 +538,13 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi archive.RecordCount = recordCount archive.BuildTime = int(dates.Since(start) / time.Millisecond) - log.WithFields(logrus.Fields{ - "record_count": recordCount, - "filename": file.Name(), - "file_size": archive.Size, - "file_hash": archive.Hash, - "elapsed": dates.Since(start), - }).Debug("completed writing archive file") + log.Debug("completed writing archive file", + "record_count", recordCount, + "filename", file.Name(), + "file_size", archive.Size, + "file_hash", archive.Hash, + "elapsed", dates.Since(start), + ) return nil } @@ -578,16 +576,15 @@ func UploadArchive(ctx context.Context, s3Client s3iface.S3API, bucket string, a archive.NeedsDeletion = true - logrus.WithFields(logrus.Fields{ - "org_id": archive.Org.ID, - "archive_type": archive.ArchiveType, - "start_date": archive.StartDate, - "period": archive.Period, - "url": archive.URL, - "file_size": archive.Size, - "file_hash": archive.Hash, - }).Debug("completed uploading archive file") - + slog.Debug("completed uploading archive file", + "org_id", archive.Org.ID, + "archive_type", archive.ArchiveType, + "start_date", archive.StartDate, + "period", archive.Period, + "url", archive.URL, + "file_size", archive.Size, + "file_hash", archive.Hash, + ) return nil } @@ -667,14 +664,14 @@ func DeleteArchiveFile(archive *Archive) error { return errors.Wrapf(err, "error deleting temp archive file: %s", archive.ArchiveFile) } - logrus.WithFields(logrus.Fields{ - "org_id": archive.Org.ID, - "archive_type": archive.ArchiveType, - "start_date": archive.StartDate, - "periond": archive.Period, - "db_archive_id": archive.ID, - "filename": archive.ArchiveFile, - }).Debug("deleted temporary archive file") + slog.Debug("deleted temporary archive file", + "org_id", archive.Org.ID, + "archive_type", archive.ArchiveType, + "start_date", archive.StartDate, + "periond", archive.Period, + "db_archive_id", archive.ID, + "filename", archive.ArchiveFile, + ) return nil } @@ -722,7 +719,7 @@ func createArchive(ctx context.Context, db *sqlx.DB, config *Config, s3Client s3 if !config.KeepFiles { err := DeleteArchiveFile(archive) if err != nil { - logrus.WithError(err).Error("error deleting temporary archive file") + slog.Error("error deleting temporary archive file", "error", err) } } }() @@ -743,21 +740,21 @@ func createArchive(ctx context.Context, db *sqlx.DB, config *Config, s3Client s3 } func createArchives(ctx context.Context, db *sqlx.DB, config *Config, s3Client s3iface.S3API, org Org, archives []*Archive) ([]*Archive, []*Archive) { - log := logrus.WithFields(logrus.Fields{"org_id": org.ID, "org_name": org.Name}) + log := slog.With("org_id", org.ID, "org_name", org.Name) created := make([]*Archive, 0, len(archives)) failed := make([]*Archive, 0, 5) for _, archive := range archives { - log.WithFields(logrus.Fields{"start_date": archive.StartDate, "end_date": archive.endDate(), "period": archive.Period, "archive_type": archive.ArchiveType}).Debug("starting archive") + log.With("start_date", archive.StartDate, "end_date", archive.endDate(), "period", archive.Period, "archive_type", archive.ArchiveType).Debug("starting archive") start := dates.Now() err := createArchive(ctx, db, config, s3Client, archive) if err != nil { - log.WithError(err).Error("error creating archive") + log.Error("error creating archive", "error", err) failed = append(failed, archive) } else { - log.WithFields(logrus.Fields{"id": archive.ID, "record_count": archive.RecordCount, "elapsed": dates.Since(start)}).Debug("archive complete") + log.Debug("archive complete", "id", archive.ID, "record_count", archive.RecordCount, "elapsed", dates.Since(start)) created = append(created, archive) } } @@ -770,7 +767,7 @@ func RollupOrgArchives(ctx context.Context, now time.Time, config *Config, db *s ctx, cancel := context.WithTimeout(ctx, time.Hour*3) defer cancel() - log := logrus.WithFields(logrus.Fields{"org_id": org.ID, "org_name": org.Name, "archive_type": archiveType}) + log := slog.With("org_id", org.ID, "org_name", org.Name, "archive_type", archiveType) // get our missing monthly archives archives, err := GetMissingMonthlyArchives(ctx, db, now, org, archiveType) @@ -783,12 +780,12 @@ func RollupOrgArchives(ctx context.Context, now time.Time, config *Config, db *s // build them from rollups for _, archive := range archives { - log := log.WithFields(logrus.Fields{"start_date": archive.StartDate}) + log := log.With("start_date", archive.StartDate) start := dates.Now() err = BuildRollupArchive(ctx, db, config, s3Client, archive, now, org, archiveType) if err != nil { - log.WithError(err).Error("error building monthly archive") + log.Error("error building monthly archive", "error", err) failed = append(failed, archive) continue } @@ -796,7 +793,7 @@ func RollupOrgArchives(ctx context.Context, now time.Time, config *Config, db *s if config.UploadToS3 { err = UploadArchive(ctx, s3Client, config.S3Bucket, archive) if err != nil { - log.WithError(err).Error("error writing archive to s3") + log.Error("error writing archive to s3", "error", err) failed = append(failed, archive) continue } @@ -804,7 +801,7 @@ func RollupOrgArchives(ctx context.Context, now time.Time, config *Config, db *s err = WriteArchiveToDB(ctx, db, archive) if err != nil { - log.WithError(err).Error("error writing record to db") + log.Error("error writing record to db", "error", err) failed = append(failed, archive) continue } @@ -812,12 +809,12 @@ func RollupOrgArchives(ctx context.Context, now time.Time, config *Config, db *s if !config.KeepFiles { err := DeleteArchiveFile(archive) if err != nil { - log.WithError(err).Error("error deleting temporary file") + log.Error("error deleting temporary file", "error", err) continue } } - log.WithFields(logrus.Fields{"id": archive.ID, "record_count": archive.RecordCount, "elapsed": dates.Since(start)}).Info("rollup created") + log.Info("rollup created", "id", archive.ID, "record_count", archive.RecordCount, "elapsed", dates.Since(start)) created = append(created, archive) } @@ -839,14 +836,14 @@ func DeleteArchivedOrgRecords(ctx context.Context, now time.Time, config *Config // for each archive deleted := make([]*Archive, 0, len(archives)) for _, a := range archives { - log := logrus.WithFields(logrus.Fields{ - "archive_id": a.ID, - "org_id": a.OrgID, - "type": a.ArchiveType, - "count": a.RecordCount, - "start": a.StartDate, - "period": a.Period, - }) + log := slog.With( + "archive_id", a.ID, + "org_id", a.OrgID, + "type", a.ArchiveType, + "count", a.RecordCount, + "start", a.StartDate, + "period", a.Period, + ) start := dates.Now() @@ -868,12 +865,12 @@ func DeleteArchivedOrgRecords(ctx context.Context, now time.Time, config *Config } if err != nil { - log.WithError(err).Error("error deleting archive") + log.Error("error deleting archive", "error", err) continue } deleted = append(deleted, a) - log.WithFields(logrus.Fields{"elapsed": dates.Since(start)}).Info("deleted archive records") + log.Info("deleted archive records", "elapsed", dates.Since(start)) } return deleted, nil @@ -881,7 +878,7 @@ func DeleteArchivedOrgRecords(ctx context.Context, now time.Time, config *Config // ArchiveOrg looks for any missing archives for the passed in org, creating and uploading them as necessary, returning the created archives func ArchiveOrg(ctx context.Context, now time.Time, cfg *Config, db *sqlx.DB, s3Client s3iface.S3API, org Org, archiveType ArchiveType) ([]*Archive, []*Archive, []*Archive, []*Archive, []*Archive, error) { - log := logrus.WithFields(logrus.Fields{"org_id": org.ID, "org_name": org.Name}) + log := slog.With("org_id", org.ID, "org_name", org.Name) start := dates.Now() dailiesCreated, dailiesFailed, monthliesCreated, monthliesFailed, err := CreateOrgArchives(ctx, now, cfg, db, s3Client, org, archiveType) @@ -892,7 +889,7 @@ func ArchiveOrg(ctx context.Context, now time.Time, cfg *Config, db *sqlx.DB, s3 if len(dailiesCreated) > 0 { elapsed := dates.Since(start) rate := float32(countRecords(dailiesCreated)) / (float32(elapsed) / float32(time.Second)) - log.WithFields(logrus.Fields{"elapsed": elapsed, "records_per_second": rate}).Info("completed archival for org") + log.Info("completed archival for org", "elapsed", elapsed, "records_per_second", rate) } rollupsCreated, rollupsFailed, err := RollupOrgArchives(ctx, now, cfg, db, s3Client, org, archiveType) @@ -939,12 +936,12 @@ func ArchiveActiveOrgs(db *sqlx.DB, cfg *Config, s3Client s3iface.S3API) error { for _, org := range orgs { // no single org should take more than 12 hours ctx, cancel := context.WithTimeout(context.Background(), time.Hour*12) - log := logrus.WithField("org_id", org.ID).WithField("org_name", org.Name) + log := slog.With("org_id", org.ID, "org_name", org.Name) if cfg.ArchiveMessages { dailiesCreated, dailiesFailed, monthliesCreated, monthliesFailed, _, err := ArchiveOrg(ctx, start, cfg, db, s3Client, org, MessageType) if err != nil { - log.WithError(err).WithField("archive_type", MessageType).Error("error archiving org messages") + log.Error("error archiving org messages", "error", err, "archive_type", MessageType) } totalMsgsRecordsArchived += countRecords(dailiesCreated) totalMsgsArchivesCreated += len(dailiesCreated) @@ -955,7 +952,7 @@ func ArchiveActiveOrgs(db *sqlx.DB, cfg *Config, s3Client s3iface.S3API) error { if cfg.ArchiveRuns { dailiesCreated, dailiesFailed, monthliesCreated, monthliesFailed, _, err := ArchiveOrg(ctx, start, cfg, db, s3Client, org, RunType) if err != nil { - log.WithError(err).WithField("archive_type", RunType).Error("error archiving org runs") + log.Error("error archiving org runs", "error", err, "archive_type", RunType) } totalRunsRecordsArchived += countRecords(dailiesCreated) totalRunsArchivesCreated += len(dailiesCreated) @@ -968,7 +965,7 @@ func ArchiveActiveOrgs(db *sqlx.DB, cfg *Config, s3Client s3iface.S3API) error { } timeTaken := dates.Now().Sub(start) - logrus.WithField("time_taken", timeTaken).WithField("num_orgs", len(orgs)).Info("archiving of active orgs complete") + slog.Info("archiving of active orgs complete", "time_taken", timeTaken, "num_orgs", len(orgs)) analytics.Gauge("archiver.archive_elapsed", timeTaken.Seconds()) analytics.Gauge("archiver.orgs_archived", float64(len(orgs))) diff --git a/archives/archives_test.go b/archives/archives_test.go index f155b1b..2f1540c 100644 --- a/archives/archives_test.go +++ b/archives/archives_test.go @@ -4,6 +4,7 @@ import ( "compress/gzip" "context" "io" + "log/slog" "os" "testing" "time" @@ -13,7 +14,6 @@ import ( "github.com/nyaruka/ezconf" "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dates" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) @@ -26,7 +26,8 @@ func setup(t *testing.T) *sqlx.DB { _, err = db.Exec(string(testDB)) assert.NoError(t, err) - logrus.SetLevel(logrus.DebugLevel) + + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}))) return db } diff --git a/archives/messages.go b/archives/messages.go index f59053a..91ce359 100644 --- a/archives/messages.go +++ b/archives/messages.go @@ -4,13 +4,13 @@ import ( "bufio" "context" "fmt" + "log/slog" "time" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/jmoiron/sqlx" "github.com/nyaruka/gocommon/dates" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) const ( @@ -93,7 +93,7 @@ func writeMessageRecords(ctx context.Context, db *sqlx.DB, archive *Archive, wri recordCount++ } - logrus.WithField("record_count", recordCount).Debug("Done Writing") + slog.Debug("Done Writing", "record_count", recordCount) return recordCount, nil } @@ -119,14 +119,14 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 defer cancel() start := dates.Now() - log := logrus.WithFields(logrus.Fields{ - "id": archive.ID, - "org_id": archive.OrgID, - "start_date": archive.StartDate, - "end_date": archive.endDate(), - "archive_type": archive.ArchiveType, - "total_count": archive.RecordCount, - }) + log := slog.With( + "id", archive.ID, + "org_id", archive.OrgID, + "start_date", archive.StartDate, + "end_date", archive.endDate(), + "archive_type", archive.ArchiveType, + "total_count", archive.RecordCount, + ) log.Info("deleting messages") // first things first, make sure our file is correct on S3 @@ -169,7 +169,7 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 } rows.Close() - log.WithField("msg_count", len(msgIDs)).Debug("found messages") + log.Debug("found messages", "msg_count", len(msgIDs)) // verify we don't see more messages than there are in our archive (fewer is ok) if visibleCount > archive.RecordCount { @@ -208,7 +208,7 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 return errors.Wrap(err, "error committing message delete transaction") } - log.WithField("elapsed", dates.Since(start)).WithField("count", len(idBatch)).Debug("deleted batch of messages") + log.Debug("deleted batch of messages", "elapsed", dates.Since(start), "count", len(idBatch)) cancel() } @@ -226,7 +226,7 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 archive.NeedsDeletion = false archive.DeletedOn = &deletedOn - logrus.WithField("elapsed", dates.Since(start)).Info("completed deleting messages") + slog.Info("completed deleting messages", "elapsed", dates.Since(start)) return nil } @@ -251,7 +251,8 @@ func DeleteBroadcasts(ctx context.Context, now time.Time, config *Config, db *sq count := 0 for rows.Next() { if count == 0 { - logrus.WithField("org_id", org.ID).Info("deleting broadcasts") + slog.Info("deleting broadcasts", "org_id", org.ID) + } // been deleting this org more than an hour? thats enough for today, exit out @@ -307,7 +308,7 @@ func DeleteBroadcasts(ctx context.Context, now time.Time, config *Config, db *sq } if count > 0 { - logrus.WithFields(logrus.Fields{"elapsed": dates.Since(start), "count": count, "org_id": org.ID}).Info("completed deleting broadcasts") + slog.Info("completed deleting broadcasts", "elapsed", dates.Since(start), "count", count, "org_id", org.ID) } return nil diff --git a/archives/runs.go b/archives/runs.go index 9a35957..f430418 100644 --- a/archives/runs.go +++ b/archives/runs.go @@ -4,13 +4,13 @@ import ( "bufio" "context" "fmt" + "log/slog" "time" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/jmoiron/sqlx" "github.com/nyaruka/gocommon/dates" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) const ( @@ -114,14 +114,14 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie defer cancel() start := dates.Now() - log := logrus.WithFields(logrus.Fields{ - "id": archive.ID, - "org_id": archive.OrgID, - "start_date": archive.StartDate, - "end_date": archive.endDate(), - "archive_type": archive.ArchiveType, - "total_count": archive.RecordCount, - }) + log := slog.With( + "id", archive.ID, + "org_id", archive.OrgID, + "start_date", archive.StartDate, + "end_date", archive.endDate(), + "archive_type", archive.ArchiveType, + "total_count", archive.RecordCount, + ) log.Info("deleting runs") // first things first, make sure our file is correct on S3 @@ -167,7 +167,7 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie } rows.Close() - log.WithField("run_count", len(runIDs)).Debug("found runs") + log.Debug("found runs", "run_count", len(runIDs)) // verify we don't see more runs than there are in our archive (fewer is ok) if runCount > archive.RecordCount { @@ -200,7 +200,7 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie return errors.Wrap(err, "error committing run delete transaction") } - log.WithField("elapsed", dates.Since(start)).WithField("count", len(idBatch)).Debug("deleted batch of runs") + log.Debug("deleted batch of runs", "elapsed", dates.Since(start), "count", len(idBatch)) cancel() } @@ -218,7 +218,7 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie archive.NeedsDeletion = false archive.DeletedOn = &deletedOn - logrus.WithField("elapsed", dates.Since(start)).Info("completed deleting runs") + slog.Info("completed deleting runs", "elapsed", dates.Since(start)) return nil } @@ -243,7 +243,7 @@ func DeleteFlowStarts(ctx context.Context, now time.Time, config *Config, db *sq count := 0 for rows.Next() { if count == 0 { - logrus.WithField("org_id", org.ID).Info("deleting starts") + slog.Info("deleting starts", "org_id", org.ID) } // been deleting this org more than an hour? thats enough for today, exit out @@ -306,7 +306,7 @@ func DeleteFlowStarts(ctx context.Context, now time.Time, config *Config, db *sq } if count > 0 { - logrus.WithFields(logrus.Fields{"elapsed": dates.Since(start), "count": count, "org_id": org.ID}).Info("completed deleting starts") + slog.Info("completed deleting starts", "elapsed", dates.Since(start), "count", count, "org_id", org.ID) } return nil diff --git a/archives/s3.go b/archives/s3.go index 5e7c3ca..17bd698 100644 --- a/archives/s3.go +++ b/archives/s3.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "log/slog" "net/url" "os" "strings" @@ -17,7 +18,6 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/sirupsen/logrus" ) const s3BucketURL = "https://%s.s3.amazonaws.com%s" @@ -44,7 +44,7 @@ func NewS3Client(config *Config) (s3iface.S3API, error) { return nil, err } s3Session.Handlers.Send.PushFront(func(r *request.Request) { - logrus.WithField("headers", r.HTTPRequest.Header).WithField("service", r.ClientInfo.ServiceName).WithField("operation", r.Operation).WithField("params", r.Params).Debug("making aws request") + slog.Debug("making aws request", "headers", r.HTTPRequest.Header, "service", r.ClientInfo.ServiceName, "operation", r.Operation, "params", r.Params) }) s3Client := s3.New(s3Session) @@ -52,7 +52,7 @@ func NewS3Client(config *Config) (s3iface.S3API, error) { // test out our S3 credentials err = TestS3(s3Client, config.S3Bucket) if err != nil { - logrus.WithError(err).Fatal("s3 bucket not reachable") + slog.Error("s3 bucket not reachable", "error", err) return nil, err } diff --git a/cmd/rp-archiver/main.go b/cmd/rp-archiver/main.go index 585cc38..d79f811 100644 --- a/cmd/rp-archiver/main.go +++ b/cmd/rp-archiver/main.go @@ -1,6 +1,7 @@ package main import ( + "log/slog" "os" "strings" "sync" @@ -14,6 +15,7 @@ import ( "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/rp-archiver/archives" + "github.com/nyaruka/rp-archiver/utils" "github.com/sirupsen/logrus" ) @@ -42,6 +44,12 @@ func main() { logrus.SetFormatter(&logrus.TextFormatter{}) logrus.WithField("version", version).WithField("released", date).Info("starting archiver") + // configure golang std structured logging to route to logrus + slog.SetDefault(slog.New(utils.NewLogrusHandler(logrus.StandardLogger()))) + + logger := slog.With("comp", "main") + logger.Info("starting archiver", "version", version, "released", date) + // if we have a DSN entry, try to initialize it if config.SentryDSN != "" { hook, err := logrus_sentry.NewSentryHook(config.SentryDSN, []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel}) @@ -57,7 +65,7 @@ func main() { // our settings shouldn't contain a timezone, nothing will work right with this not being a constant UTC if strings.Contains(config.DB, "TimeZone") { - logrus.WithField("db", config.DB).Fatalf("invalid db connection string, do not specify a timezone, archiver always uses UTC") + logger.Error("invalid db connection string, do not specify a timezone, archiver always uses UTC", "db", config.DB) } // force our DB connection to be in UTC @@ -69,19 +77,19 @@ func main() { db, err := sqlx.Open("postgres", config.DB) if err != nil { - logrus.Fatal(err) + logger.Error("error connecting to db", "error", err) } else { db.SetMaxOpenConns(2) - logrus.WithField("state", "starting").Info("db ok") + logger.Info("db ok", "state", "starting") } var s3Client s3iface.S3API if config.UploadToS3 { s3Client, err = archives.NewS3Client(config) if err != nil { - logrus.WithError(err).Fatal("unable to initialize s3 client") + logger.Error("unable to initialize s3 client", "error", err) } else { - logrus.WithField("state", "starting").Info("s3 bucket ok") + logger.Info("s3 bucket ok", "state", "starting") } } @@ -90,15 +98,15 @@ func main() { // ensure that we can actually write to the temp directory err = archives.EnsureTempArchiveDirectory(config.TempDir) if err != nil { - logrus.WithError(err).Fatal("cannot write to temp directory") + logger.Error("cannot write to temp directory", "error", err) } else { - logrus.WithField("state", "starting").Info("tmp file access ok") + logger.Info("tmp file access ok", "state", "starting") } // parse our start time timeOfDay, err := dates.ParseTimeOfDay("tt:mm", config.StartTime) if err != nil { - logrus.WithError(err).Fatal("invalid start time supplied, format: HH:MM") + logger.Error("invalid start time supplied, format: HH:MM", "error", err) } // if we have a librato token, configure it @@ -115,7 +123,7 @@ func main() { nextArchival := getNextArchivalTime(timeOfDay) napTime := time.Until(nextArchival) - logrus.WithField("sleep_time", napTime).WithField("next_archival", nextArchival).Info("sleeping until next archival") + logger.Info("sleeping until next archival", "sleep_time", napTime, "next_archival", nextArchival) time.Sleep(napTime) doArchival(db, config, s3Client) @@ -131,7 +139,7 @@ func doArchival(db *sqlx.DB, cfg *archives.Config, s3Client s3iface.S3API) { // try to archive all active orgs, and if it fails, wait 5 minutes and try again err := archives.ArchiveActiveOrgs(db, cfg, s3Client) if err != nil { - logrus.WithError(err).Error("error archiving, will retry in 5 minutes") + slog.Error("error archiving, will retry in 5 minutes", "error", err) time.Sleep(time.Minute * 5) continue } else { diff --git a/utils/logrus.go b/utils/logrus.go new file mode 100644 index 0000000..2650bf9 --- /dev/null +++ b/utils/logrus.go @@ -0,0 +1,92 @@ +// Structured logging handler for logrus so we can rewrite code to use slog package incrementally. Once all logging is +// happening via slog, we just need to hook up Sentry directly to that, and then we can get rid of this file. +package utils + +import ( + "context" + "log/slog" + "slices" + "strings" + + "github.com/sirupsen/logrus" +) + +var levels = map[slog.Level]logrus.Level{ + slog.LevelError: logrus.ErrorLevel, + slog.LevelWarn: logrus.WarnLevel, + slog.LevelInfo: logrus.InfoLevel, + slog.LevelDebug: logrus.DebugLevel, +} + +type LogrusHandler struct { + logger *logrus.Logger + groups []string + attrs []slog.Attr +} + +func NewLogrusHandler(logger *logrus.Logger) *LogrusHandler { + return &LogrusHandler{logger: logger} +} + +func (l *LogrusHandler) clone() *LogrusHandler { + return &LogrusHandler{ + logger: l.logger, + groups: slices.Clip(l.groups), + attrs: slices.Clip(l.attrs), + } +} + +func (l *LogrusHandler) Enabled(ctx context.Context, level slog.Level) bool { + return levels[level] <= l.logger.GetLevel() +} + +func (l *LogrusHandler) Handle(ctx context.Context, r slog.Record) error { + log := logrus.NewEntry(l.logger) + if r.Time.IsZero() { + log = log.WithTime(r.Time) + } + + f := logrus.Fields{} + for _, a := range l.attrs { + if a.Key != "" { + f[a.Key] = a.Value + } + } + log = log.WithFields(f) + + r.Attrs(func(attr slog.Attr) bool { + if attr.Key == "" { + return true + } + log = log.WithField(attr.Key, attr.Value) + return true + }) + log.Logf(levels[r.Level], r.Message) + return nil +} + +func (l *LogrusHandler) groupPrefix() string { + if len(l.groups) > 0 { + return strings.Join(l.groups, ":") + ":" + } + return "" +} + +func (l *LogrusHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + newHandler := l.clone() + for _, a := range attrs { + newHandler.attrs = append(newHandler.attrs, slog.Attr{ + Key: l.groupPrefix() + a.Key, + Value: a.Value, + }) + } + return newHandler +} + +func (l *LogrusHandler) WithGroup(name string) slog.Handler { + newHandler := l.clone() + newHandler.groups = append(newHandler.groups, name) + return newHandler +} + +var _ slog.Handler = &LogrusHandler{} From d12414fb0121e0b076a9b202b50229cc34cdf566 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 11 Apr 2024 15:16:26 -0500 Subject: [PATCH 04/18] Remove flows_flowrun.submitted_by --- archives/archives_test.go | 12 ++++++------ archives/runs.go | 9 +++------ archives/testdata/runs1.jsonl | 4 ++-- archives/testdata/runs2.jsonl | 2 +- testdb.sql | 17 ++++++++--------- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/archives/archives_test.go b/archives/archives_test.go index f155b1b..d7718a5 100644 --- a/archives/archives_test.go +++ b/archives/archives_test.go @@ -223,8 +223,8 @@ func TestCreateRunArchive(t *testing.T) { // should have two record assert.Equal(t, 2, task.RecordCount) - assert.Equal(t, int64(472), task.Size) - assert.Equal(t, "734d437e1c66d09e033d698c732178f8", task.Hash) + assert.Equal(t, int64(458), task.Size) + assert.Equal(t, "7220a13c19f5b6065e7d4c419c114635", task.Hash) assertArchiveFile(t, task, "runs1.jsonl") DeleteArchiveFile(task) @@ -243,8 +243,8 @@ func TestCreateRunArchive(t *testing.T) { // should have one record assert.Equal(t, 1, task.RecordCount) - assert.Equal(t, int64(490), task.Size) - assert.Equal(t, "c2138e3c3009a9c09fc55482903d93e4", task.Hash) + assert.Equal(t, int64(465), task.Size) + assert.Equal(t, "40abf2113ea7c25c5476ff3025d54b07", task.Hash) assertArchiveFile(t, task, "runs2.jsonl") DeleteArchiveFile(task) @@ -449,10 +449,10 @@ func TestArchiveOrgRuns(t *testing.T) { assert.Equal(t, 10, len(dailiesCreated)) assertArchive(t, dailiesCreated[0], time.Date(2017, 10, 1, 0, 0, 0, 0, time.UTC), DayPeriod, 0, 23, "f0d79988b7772c003d04a28bd7417a62") - assertArchive(t, dailiesCreated[9], time.Date(2017, 10, 10, 0, 0, 0, 0, time.UTC), DayPeriod, 2, 1984, "869cc00ad4cca0371d07c88d8cf2bf26") + assertArchive(t, dailiesCreated[9], time.Date(2017, 10, 10, 0, 0, 0, 0, time.UTC), DayPeriod, 2, 1953, "95475b968ceff15f2f90d539e1bd3d20") assert.Equal(t, 2, len(monthliesCreated)) - assertArchive(t, monthliesCreated[0], time.Date(2017, 8, 1, 0, 0, 0, 0, time.UTC), MonthPeriod, 1, 490, "c2138e3c3009a9c09fc55482903d93e4") + assertArchive(t, monthliesCreated[0], time.Date(2017, 8, 1, 0, 0, 0, 0, time.UTC), MonthPeriod, 1, 465, "40abf2113ea7c25c5476ff3025d54b07") assertArchive(t, monthliesCreated[1], time.Date(2017, 9, 1, 0, 0, 0, 0, time.UTC), MonthPeriod, 0, 23, "f0d79988b7772c003d04a28bd7417a62") assert.Equal(t, 12, len(deleted)) diff --git a/archives/runs.go b/archives/runs.go index 9a35957..46e76fa 100644 --- a/archives/runs.go +++ b/archives/runs.go @@ -48,14 +48,11 @@ FROM ( WHEN status = 'X' THEN 'expired' WHEN status = 'F' THEN 'failed' ELSE NULL - END as exit_type, - a.username as submitted_by + END as exit_type FROM flows_flowrun fr - LEFT JOIN auth_user a ON a.id = fr.submitted_by_id - JOIN LATERAL (SELECT uuid, name FROM flows_flow WHERE flows_flow.id = fr.flow_id) AS flow_struct ON True - JOIN LATERAL (SELECT uuid, name FROM contacts_contact cc WHERE cc.id = fr.contact_id) AS contact_struct ON True - + JOIN LATERAL (SELECT uuid, name FROM flows_flow WHERE flows_flow.id = fr.flow_id) AS flow_struct ON True + JOIN LATERAL (SELECT uuid, name FROM contacts_contact cc WHERE cc.id = fr.contact_id) AS contact_struct ON True WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3 ORDER BY fr.modified_on ASC, id ASC ) as rec;` diff --git a/archives/testdata/runs1.jsonl b/archives/testdata/runs1.jsonl index 3e7f8ea..51f81dd 100644 --- a/archives/testdata/runs1.jsonl +++ b/archives/testdata/runs1.jsonl @@ -1,2 +1,2 @@ -{"id":1,"uuid":"4ced1260-9cfe-4b7f-81dd-b637108f15b9","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[],"values":{},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed","submitted_by":null} -{"id":2,"uuid":"7d68469c-0494-498a-bdf3-bac68321fd6d","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[{"node": "10896d63-8df7-4022-88dd-a9d93edf355b", "time": "2017-08-12T13:07:24.049815+00:00"}],"values":{"agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed","submitted_by":null} +{"id":1,"uuid":"4ced1260-9cfe-4b7f-81dd-b637108f15b9","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[],"values":{},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"} +{"id":2,"uuid":"7d68469c-0494-498a-bdf3-bac68321fd6d","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[{"node": "10896d63-8df7-4022-88dd-a9d93edf355b", "time": "2017-08-12T13:07:24.049815+00:00"}],"values":{"agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"} diff --git a/archives/testdata/runs2.jsonl b/archives/testdata/runs2.jsonl index 3c82b3c..cfd6714 100644 --- a/archives/testdata/runs2.jsonl +++ b/archives/testdata/runs2.jsonl @@ -1 +1 @@ -{"id":3,"uuid":"de782b35-a398-46ed-8550-34c66053841b","flow":{"uuid":"629db399-a5fb-4fa0-88e6-f479957b63d2","name":"Flow 2"},"contact":{"uuid":"7051dff0-0a27-49d7-af1f-4494239139e6","name":"Joanne Stone"},"responded":true,"path":[{"node": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "time": "2017-08-12T13:07:24.049815+00:00"}],"values":{"agree": {"name": "Agree", "node": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}, "confirm_agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ab", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Confirmed Strongly agree"}},"created_on":"2017-08-10T19:11:59.890662+00:00","modified_on":"2017-08-10T19:11:59.890662+00:00","exited_on":"2017-08-10T19:11:59.890662+00:00","exit_type":"completed","submitted_by":"greg@gmail.com"} +{"id":3,"uuid":"de782b35-a398-46ed-8550-34c66053841b","flow":{"uuid":"629db399-a5fb-4fa0-88e6-f479957b63d2","name":"Flow 2"},"contact":{"uuid":"7051dff0-0a27-49d7-af1f-4494239139e6","name":"Joanne Stone"},"responded":true,"path":[{"node": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "time": "2017-08-12T13:07:24.049815+00:00"}],"values":{"agree": {"name": "Agree", "node": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}, "confirm_agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ab", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Confirmed Strongly agree"}},"created_on":"2017-08-10T19:11:59.890662+00:00","modified_on":"2017-08-10T19:11:59.890662+00:00","exited_on":"2017-08-10T19:11:59.890662+00:00","exit_type":"completed"} diff --git a/testdb.sql b/testdb.sql index d823ba6..a0513ca 100644 --- a/testdb.sql +++ b/testdb.sql @@ -212,7 +212,6 @@ CREATE TABLE flows_flowrun ( created_on timestamp with time zone NOT NULL, modified_on timestamp with time zone NOT NULL, exited_on timestamp with time zone NULL, - submitted_by_id integer NULL REFERENCES auth_user(id), status varchar(1) NOT NULL, delete_from_results boolean ); @@ -336,23 +335,23 @@ INSERT INTO flows_flowstart_groups(flowstart_id, contactgroup_id) VALUES INSERT INTO flows_flowstart_calls(flowstart_id, call_id) VALUES (1, 1); -INSERT INTO flows_flowrun(id, uuid, org_id, responded, contact_id, flow_id, results, path, created_on, modified_on, exited_on, status, submitted_by_id, start_id) VALUES -(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', 2, TRUE, 6, 1, '{}', '[]', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL, 1), +INSERT INTO flows_flowrun(id, uuid, org_id, responded, contact_id, flow_id, results, path, created_on, modified_on, exited_on, status, start_id) VALUES +(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', 2, TRUE, 6, 1, '{}', '[]', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', 1), (2, '7d68469c-0494-498a-bdf3-bac68321fd6d', 2, TRUE, 6, 1, '{"agree": {"category": "Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}', '[{"uuid": "c3d0b417-db75-417c-8050-33776ec8f620", "node_uuid": "10896d63-8df7-4022-88dd-a9d93edf355b", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "2f890507-2ad2-4bd1-92fc-0ca031155fca"}]', -'2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL, NULL), +'2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL), (3, 'de782b35-a398-46ed-8550-34c66053841b', 3, TRUE, 7, 2, '{"agree": {"category": "Strongly agree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}, "confirm_agree": {"category": "Confirmed Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ab", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}', '[{"uuid": "600ac5b4-4895-4161-ad97-6e2f1bb48bcb", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', -'2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', 1, NULL), +'2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', NULL), (4, '329a5d24-64fc-479c-8d24-9674c9b46530', 3, TRUE, 7, 2, '{"agree": {"category": "Disagree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "B", "created_on": "2017-10-10T12:25:21.714339+00:00", "input": "B"}}', '[{"uuid": "babf4fc8-e12c-4bb9-a9dd-61178a118b5a", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-10-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', -'2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL, NULL), -(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', 3, TRUE, 7, 2, '{}', '[]', '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL, NULL), -(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', 3, TRUE, 7, 2, '{}', '[]', '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL, NULL), -(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', 2, TRUE, 7, 2, '{}', '[]', '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL, NULL); +'2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), +(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', 3, TRUE, 7, 2, '{}', '[]', '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), +(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', 3, TRUE, 7, 2, '{}', '[]', '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL), +(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', 2, TRUE, 7, 2, '{}', '[]', '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL); -- update run #5 to have a path longer than 500 steps UPDATE flows_flowrun SET path = s.path FROM ( From 32e48d81614b5bca019ca4436a67dfa3898a0327 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 13 Oct 2023 18:01:14 +0200 Subject: [PATCH 05/18] Finish switching to use slog for logging --- cmd/rp-archiver/main.go | 49 +++++++++++++--------- go.mod | 10 ++--- go.sum | 36 +++++++--------- utils/logrus.go | 92 ----------------------------------------- 4 files changed, 50 insertions(+), 137 deletions(-) delete mode 100644 utils/logrus.go diff --git a/cmd/rp-archiver/main.go b/cmd/rp-archiver/main.go index d79f811..d2ff793 100644 --- a/cmd/rp-archiver/main.go +++ b/cmd/rp-archiver/main.go @@ -1,6 +1,7 @@ package main import ( + "log" "log/slog" "os" "strings" @@ -8,15 +9,15 @@ import ( "time" "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/evalphobia/logrus_sentry" + "github.com/getsentry/sentry-go" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" "github.com/nyaruka/ezconf" "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/rp-archiver/archives" - "github.com/nyaruka/rp-archiver/utils" - "github.com/sirupsen/logrus" + slogmulti "github.com/samber/slog-multi" + slogsentry "github.com/samber/slog-sentry" ) var ( @@ -31,36 +32,44 @@ func main() { loader.MustLoad() if config.KeepFiles && !config.UploadToS3 { - logrus.Fatal("cannot delete archives and also not upload to s3") + log.Fatal("cannot delete archives and also not upload to s3") } - level, err := logrus.ParseLevel(config.LogLevel) + var level slog.Level + err := level.UnmarshalText([]byte(config.LogLevel)) if err != nil { - logrus.Fatalf("Invalid log level '%s'", level) + log.Fatalf("invalid log level %s", level) + os.Exit(1) } - logrus.SetLevel(level) - logrus.SetOutput(os.Stdout) - logrus.SetFormatter(&logrus.TextFormatter{}) - logrus.WithField("version", version).WithField("released", date).Info("starting archiver") - - // configure golang std structured logging to route to logrus - slog.SetDefault(slog.New(utils.NewLogrusHandler(logrus.StandardLogger()))) + // configure our logger + logHandler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level}) + slog.SetDefault(slog.New(logHandler)) logger := slog.With("comp", "main") logger.Info("starting archiver", "version", version, "released", date) // if we have a DSN entry, try to initialize it if config.SentryDSN != "" { - hook, err := logrus_sentry.NewSentryHook(config.SentryDSN, []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel}) - hook.Timeout = 0 - hook.StacktraceConfiguration.Enable = true - hook.StacktraceConfiguration.Skip = 4 - hook.StacktraceConfiguration.Context = 5 + err := sentry.Init(sentry.ClientOptions{ + Dsn: config.SentryDSN, + EnableTracing: false, + }) if err != nil { - logrus.Fatalf("invalid sentry DSN: '%s': %s", config.SentryDSN, err) + log.Fatalf("error initiating sentry client, error %s, dsn %s", err, config.SentryDSN) + os.Exit(1) } - logrus.StandardLogger().Hooks.Add(hook) + + defer sentry.Flush(2 * time.Second) + + logger = slog.New( + slogmulti.Fanout( + logHandler, + slogsentry.Option{Level: slog.LevelError}.NewSentryHandler(), + ), + ) + logger = logger.With("release", version) + slog.SetDefault(logger) } // our settings shouldn't contain a timezone, nothing will work right with this not being a constant UTC diff --git a/go.mod b/go.mod index fcc34de..e3002cb 100644 --- a/go.mod +++ b/go.mod @@ -4,21 +4,20 @@ go 1.22 require ( github.com/aws/aws-sdk-go v1.51.20 - github.com/evalphobia/logrus_sentry v0.8.2 + github.com/getsentry/sentry-go v0.22.0 github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.3.0 github.com/nyaruka/gocommon v1.53.2 github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.9.3 + github.com/samber/slog-multi v1.0.2 + github.com/samber/slog-sentry v1.2.2 github.com/stretchr/testify v1.9.0 ) require ( - github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/structs v1.1.0 // indirect - github.com/getsentry/raven-go v0.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect @@ -27,7 +26,8 @@ require ( github.com/nyaruka/null/v2 v2.0.3 // indirect github.com/nyaruka/phonenumbers v1.3.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect + github.com/samber/lo v1.38.1 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index 2a84d2f..51f1390 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,18 @@ github.com/aws/aws-sdk-go v1.51.20 h1:ziM90ujYHKKkoTZL+Wg2LwjbQecL+l298GGJeG4ktZs= github.com/aws/aws-sdk-go v1.51.20/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= -github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFetyN46VoDQ= -github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluWaP6dM= +github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -42,27 +40,25 @@ github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA= github.com/nyaruka/phonenumbers v1.3.4 h1:bF1Wdh++fxw09s3surhVeBhXEcUKG07pHeP8HQXqjn8= github.com/nyaruka/phonenumbers v1.3.4/go.mod h1:Ut+eFwikULbmCenH6InMKL9csUNLyxHuBLyfkpum11s= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/slog-multi v1.0.2 h1:6BVH9uHGAsiGkbbtQgAOQJMpKgV8unMrHhhJaw+X1EQ= +github.com/samber/slog-multi v1.0.2/go.mod h1:uLAvHpGqbYgX4FSL0p1ZwoLuveIAJvBECtE07XmYvFo= +github.com/samber/slog-sentry v1.2.2 h1:S0glIVITlGCCfSvIOte2Sh63HMHJpYN3hDr+97hILIk= +github.com/samber/slog-sentry v1.2.2/go.mod h1:bHm8jm1dks0p+xc/lH2i4TIFwnPcMTvZeHgCBj5+uhA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= -golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= -golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -71,8 +67,8 @@ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGm google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/utils/logrus.go b/utils/logrus.go deleted file mode 100644 index 2650bf9..0000000 --- a/utils/logrus.go +++ /dev/null @@ -1,92 +0,0 @@ -// Structured logging handler for logrus so we can rewrite code to use slog package incrementally. Once all logging is -// happening via slog, we just need to hook up Sentry directly to that, and then we can get rid of this file. -package utils - -import ( - "context" - "log/slog" - "slices" - "strings" - - "github.com/sirupsen/logrus" -) - -var levels = map[slog.Level]logrus.Level{ - slog.LevelError: logrus.ErrorLevel, - slog.LevelWarn: logrus.WarnLevel, - slog.LevelInfo: logrus.InfoLevel, - slog.LevelDebug: logrus.DebugLevel, -} - -type LogrusHandler struct { - logger *logrus.Logger - groups []string - attrs []slog.Attr -} - -func NewLogrusHandler(logger *logrus.Logger) *LogrusHandler { - return &LogrusHandler{logger: logger} -} - -func (l *LogrusHandler) clone() *LogrusHandler { - return &LogrusHandler{ - logger: l.logger, - groups: slices.Clip(l.groups), - attrs: slices.Clip(l.attrs), - } -} - -func (l *LogrusHandler) Enabled(ctx context.Context, level slog.Level) bool { - return levels[level] <= l.logger.GetLevel() -} - -func (l *LogrusHandler) Handle(ctx context.Context, r slog.Record) error { - log := logrus.NewEntry(l.logger) - if r.Time.IsZero() { - log = log.WithTime(r.Time) - } - - f := logrus.Fields{} - for _, a := range l.attrs { - if a.Key != "" { - f[a.Key] = a.Value - } - } - log = log.WithFields(f) - - r.Attrs(func(attr slog.Attr) bool { - if attr.Key == "" { - return true - } - log = log.WithField(attr.Key, attr.Value) - return true - }) - log.Logf(levels[r.Level], r.Message) - return nil -} - -func (l *LogrusHandler) groupPrefix() string { - if len(l.groups) > 0 { - return strings.Join(l.groups, ":") + ":" - } - return "" -} - -func (l *LogrusHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - newHandler := l.clone() - for _, a := range attrs { - newHandler.attrs = append(newHandler.attrs, slog.Attr{ - Key: l.groupPrefix() + a.Key, - Value: a.Value, - }) - } - return newHandler -} - -func (l *LogrusHandler) WithGroup(name string) slog.Handler { - newHandler := l.clone() - newHandler.groups = append(newHandler.groups, name) - return newHandler -} - -var _ slog.Handler = &LogrusHandler{} From 3c3a6ea1ac23ec8e258d12c9a4c2c75649f54bd8 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 12 Apr 2024 14:54:10 -0500 Subject: [PATCH 06/18] Update CHANGELOG.md for v9.1.0 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce449a9..f3961a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v9.1.0 (2024-04-12) +------------------------- + * Remove flows_flowrun.submitted_by + * Replace logrus with slog + * Update to go 1.22 and update deps + v9.0.0 (2024-01-05) ------------------------- * Update dependencies From cf7d1b0331b68e5f2fcf59c4a085e5f62e951cb3 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 25 Apr 2024 08:09:08 -0500 Subject: [PATCH 07/18] Add support for status=READ --- archives/messages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archives/messages.go b/archives/messages.go index 91ce359..647b211 100644 --- a/archives/messages.go +++ b/archives/messages.go @@ -44,7 +44,7 @@ SELECT rec.visibility, row_to_json(rec) FROM ( WHEN status = 'E' THEN 'errored' WHEN status = 'F' THEN 'failed' WHEN status = 'S' THEN 'sent' - WHEN status = 'R' THEN 'resent' + WHEN status = 'R' THEN 'read' ELSE NULL END AS status, CASE WHEN visibility = 'V' THEN 'visible' WHEN visibility = 'A' THEN 'archived' WHEN visibility = 'D' THEN 'deleted' WHEN visibility = 'X' THEN 'deleted' ELSE NULL END as visibility, From f4497372bf2d9544af99f18f92121d0c708aab85 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 25 Apr 2024 08:49:44 -0500 Subject: [PATCH 08/18] Update CHANGELOG.md for v9.1.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3961a7..6e1ab34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v9.1.1 (2024-04-25) +------------------------- + * Add support for status=READ + v9.1.0 (2024-04-12) ------------------------- * Remove flows_flowrun.submitted_by From 79fe207d7721b99c31814f291d066a7d297b7701 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 16 May 2024 11:21:54 +0200 Subject: [PATCH 09/18] Update Github actions to latest versions --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62da6ba..7236bc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: ${{ env.go-version }} @@ -35,7 +35,7 @@ jobs: - name: Upload coverage if: success() - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true @@ -47,17 +47,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: ${{ env.go-version }} - name: Publish release - uses: goreleaser/goreleaser-action@v4 + uses: goreleaser/goreleaser-action@v5 with: args: release --clean env: From 6f98234779b1cb6b57f36836ba8c8ac37a19fa6a Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 30 May 2024 16:04:20 +0200 Subject: [PATCH 10/18] Use std lib errors --- archives/archives.go | 79 ++++++++++++++++++++++---------------------- archives/messages.go | 27 ++++++++------- archives/runs.go | 27 ++++++++------- go.mod | 2 +- 4 files changed, 66 insertions(+), 69 deletions(-) diff --git a/archives/archives.go b/archives/archives.go index a7c65f9..5a22cb2 100644 --- a/archives/archives.go +++ b/archives/archives.go @@ -19,7 +19,6 @@ import ( "github.com/lib/pq" "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dates" - "github.com/pkg/errors" ) // ArchiveType is the type for the archives @@ -104,7 +103,7 @@ func GetActiveOrgs(ctx context.Context, db *sqlx.DB, conf *Config) ([]Org, error rows, err := db.QueryxContext(ctx, sqlLookupActiveOrgs) if err != nil { - return nil, errors.Wrapf(err, "error fetching active orgs") + return nil, fmt.Errorf("error fetching active orgs: %w", err) } defer rows.Close() @@ -113,7 +112,7 @@ func GetActiveOrgs(ctx context.Context, db *sqlx.DB, conf *Config) ([]Org, error org := Org{RetentionPeriod: conf.RetentionPeriod} err = rows.StructScan(&org) if err != nil { - return nil, errors.Wrapf(err, "error scanning active org") + return nil, fmt.Errorf("error scanning active org: %w", err) } orgs = append(orgs, org) } @@ -135,7 +134,7 @@ func GetCurrentArchives(ctx context.Context, db *sqlx.DB, org Org, archiveType A archives := make([]*Archive, 0, 1) err := db.SelectContext(ctx, &archives, sqlLookupOrgArchives, org.ID, archiveType) if err != nil && err != sql.ErrNoRows { - return nil, errors.Wrapf(err, "error selecting current archives for org: %d and type: %s", org.ID, archiveType) + return nil, fmt.Errorf("error selecting current archives for org: %d and type: %s: %w", org.ID, archiveType, err) } return archives, nil @@ -155,7 +154,7 @@ func GetArchivesNeedingDeletion(ctx context.Context, db *sqlx.DB, org Org, archi archives := make([]*Archive, 0, 1) err := db.SelectContext(ctx, &archives, sqlLookupArchivesNeedingDeletion, org.ID, archiveType) if err != nil && err != sql.ErrNoRows { - return nil, errors.Wrapf(err, "error selecting archives needing deletion for org: %d and type: %s", org.ID, archiveType) + return nil, fmt.Errorf("error selecting archives needing deletion for org: %d and type: %s: %w", org.ID, archiveType, err) } return archives, nil @@ -175,7 +174,7 @@ func GetCurrentArchiveCount(ctx context.Context, db *sqlx.DB, org Org, archiveTy err := db.GetContext(ctx, &archiveCount, sqlCountOrgArchives, org.ID, archiveType) if err != nil { - return 0, errors.Wrapf(err, "error querying archive count for org: %d and type: %s", org.ID, archiveType) + return 0, fmt.Errorf("error querying archive count for org: %d and type: %s: %w", org.ID, archiveType, err) } return archiveCount, nil @@ -197,7 +196,7 @@ func GetDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, org Org, arc err := db.SelectContext(ctx, &existingArchives, sqlLookupOrgDailyArchivesForDateRange, org.ID, archiveType, DayPeriod, startDate, endDate) if err != nil && err != sql.ErrNoRows { - return nil, errors.Wrapf(err, "error selecting daily archives for org: %d and type: %s", org.ID, archiveType) + return nil, fmt.Errorf("error selecting daily archives for org: %d and type: %s: %w", org.ID, archiveType, err) } return existingArchives, nil @@ -241,7 +240,7 @@ func GetMissingDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, start rows, err := db.QueryxContext(ctx, sqlLookupMissingDailyArchive, startDate, endDate, org.ID, DayPeriod, archiveType) if err != nil { - return nil, errors.Wrapf(err, "error getting missing daily archives for org: %d and type: %s", org.ID, archiveType) + return nil, fmt.Errorf("error getting missing daily archives for org: %d and type: %s: %w", org.ID, archiveType, err) } defer rows.Close() @@ -250,7 +249,7 @@ func GetMissingDailyArchivesForDateRange(ctx context.Context, db *sqlx.DB, start err = rows.Scan(&missingDay) if err != nil { - return nil, errors.Wrapf(err, "error scanning missing daily archive for org: %d and type: %s", org.ID, archiveType) + return nil, fmt.Errorf("error scanning missing daily archive for org: %d and type: %s: %w", org.ID, archiveType, err) } archive := Archive{ Org: org, @@ -295,7 +294,7 @@ func GetMissingMonthlyArchives(ctx context.Context, db *sqlx.DB, now time.Time, rows, err := db.QueryxContext(ctx, sqlLookupMissingMonthlyArchive, startDate, endDate, org.ID, MonthPeriod, archiveType) if err != nil { - return nil, errors.Wrapf(err, "error getting missing monthly archive for org: %d and type: %s", org.ID, archiveType) + return nil, fmt.Errorf("error getting missing monthly archive for org: %d and type: %s: %w", org.ID, archiveType, err) } defer rows.Close() @@ -304,7 +303,7 @@ func GetMissingMonthlyArchives(ctx context.Context, db *sqlx.DB, now time.Time, err = rows.Scan(&missingMonth) if err != nil { - return nil, errors.Wrapf(err, "error scanning missing monthly archive for org: %d and type: %s", org.ID, archiveType) + return nil, fmt.Errorf("error scanning missing monthly archive for org: %d and type: %s: %w", org.ID, archiveType, err) } archive := Archive{ Org: org, @@ -349,7 +348,7 @@ func BuildRollupArchive(ctx context.Context, db *sqlx.DB, conf *Config, s3Client filename := fmt.Sprintf("%s_%d_%s_%d_%02d_", monthlyArchive.ArchiveType, monthlyArchive.Org.ID, monthlyArchive.Period, monthlyArchive.StartDate.Year(), monthlyArchive.StartDate.Month()) file, err := os.CreateTemp(conf.TempDir, filename) if err != nil { - return errors.Wrapf(err, "error creating temp file: %s", filename) + return fmt.Errorf("error creating temp file: %s: %w", filename, err) } writerHash := md5.New() gzWriter := gzip.NewWriter(io.MultiWriter(file, writerHash)) @@ -378,7 +377,7 @@ func BuildRollupArchive(ctx context.Context, db *sqlx.DB, conf *Config, s3Client reader, err := GetS3File(ctx, s3Client, daily.URL) if err != nil { - return errors.Wrapf(err, "error reading S3 URL: %s", daily.URL) + return fmt.Errorf("error reading S3 URL: %s: %w", daily.URL, err) } // set up our reader to calculate our hash along the way @@ -386,13 +385,13 @@ func BuildRollupArchive(ctx context.Context, db *sqlx.DB, conf *Config, s3Client teeReader := io.TeeReader(reader, readerHash) gzipReader, err := gzip.NewReader(teeReader) if err != nil { - return errors.Wrapf(err, "error creating gzip reader") + return fmt.Errorf("error creating gzip reader: %w", err) } // copy this daily file (uncompressed) to our new monthly file _, err = io.Copy(writer, gzipReader) if err != nil { - return errors.Wrapf(err, "error copying from s3 to disk for URL: %s", daily.URL) + return fmt.Errorf("error copying from s3 to disk for URL: %s: %w", daily.URL, err) } reader.Close() @@ -422,7 +421,7 @@ func BuildRollupArchive(ctx context.Context, db *sqlx.DB, conf *Config, s3Client monthlyArchive.Hash = hex.EncodeToString(writerHash.Sum(nil)) stat, err := file.Stat() if err != nil { - return errors.Wrapf(err, "error statting file: %s", file.Name()) + return fmt.Errorf("error statting file: %s: %w", file.Name(), err) } monthlyArchive.Size = stat.Size() monthlyArchive.RecordCount = recordCount @@ -444,7 +443,7 @@ func EnsureTempArchiveDirectory(path string) error { if os.IsNotExist(err) { return os.MkdirAll(path, 0700) } else if err != nil { - return errors.Wrapf(err, "error statting temp dir: %s", path) + return fmt.Errorf("error statting temp dir: %s: %w", path, err) } // is path a directory @@ -482,7 +481,7 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi filename := fmt.Sprintf("%s_%d_%s%d%02d%02d_", archive.ArchiveType, archive.Org.ID, archive.Period, archive.StartDate.Year(), archive.StartDate.Month(), archive.StartDate.Day()) file, err := os.CreateTemp(archivePath, filename) if err != nil { - return errors.Wrapf(err, "error creating temp file: %s", filename) + return fmt.Errorf("error creating temp file: %s: %w", filename, err) } defer func() { @@ -513,24 +512,24 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi } if err != nil { - return errors.Wrapf(err, "error writing archive") + return fmt.Errorf("error writing archive: %w", err) } err = writer.Flush() if err != nil { - return errors.Wrapf(err, "error flushing archive file") + return fmt.Errorf("error flushing archive file: %w", err) } err = gzWriter.Close() if err != nil { - return errors.Wrapf(err, "error closing archive gzip writer") + return fmt.Errorf("error closing archive gzip writer: %w", err) } // calculate our size and hash archive.Hash = hex.EncodeToString(hash.Sum(nil)) stat, err := file.Stat() if err != nil { - return errors.Wrapf(err, "error calculating archive hash") + return fmt.Errorf("error calculating archive hash: %w", err) } archive.ArchiveFile = file.Name() @@ -571,7 +570,7 @@ func UploadArchive(ctx context.Context, s3Client s3iface.S3API, bucket string, a err := UploadToS3(ctx, s3Client, bucket, archivePath, archive) if err != nil { - return errors.Wrapf(err, "error uploading archive to S3") + return fmt.Errorf("error uploading archive to S3: %w", err) } archive.NeedsDeletion = true @@ -603,20 +602,20 @@ func WriteArchiveToDB(ctx context.Context, db *sqlx.DB, archive *Archive) error tx, err := db.BeginTxx(ctx, nil) if err != nil { - return errors.Wrapf(err, "error starting transaction") + return fmt.Errorf("error starting transaction: %w", err) } rows, err := tx.NamedQuery(sqlInsertArchive, archive) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error inserting archive") + return fmt.Errorf("error inserting archive: %w", err) } rows.Next() err = rows.Scan(&archive.ID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error reading new archive id") + return fmt.Errorf("error reading new archive id: %w", err) } rows.Close() @@ -631,12 +630,12 @@ func WriteArchiveToDB(ctx context.Context, db *sqlx.DB, archive *Archive) error result, err := tx.ExecContext(ctx, `UPDATE archives_archive SET rollup_id = $1 WHERE id = ANY($2)`, archive.ID, pq.Array(childIDs)) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error updating rollup ids") + return fmt.Errorf("error updating rollup ids: %w", err) } affected, err := result.RowsAffected() if err != nil { tx.Rollback() - return errors.Wrapf(err, "error getting number of rollup ids updated") + return fmt.Errorf("error getting number of rollup ids updated: %w", err) } if int(affected) != len(childIDs) { tx.Rollback() @@ -647,7 +646,7 @@ func WriteArchiveToDB(ctx context.Context, db *sqlx.DB, archive *Archive) error err = tx.Commit() if err != nil { tx.Rollback() - return errors.Wrapf(err, "error committing new archive transaction") + return fmt.Errorf("error committing new archive transaction: %w", err) } return nil } @@ -661,7 +660,7 @@ func DeleteArchiveFile(archive *Archive) error { err := os.Remove(archive.ArchiveFile) if err != nil { - return errors.Wrapf(err, "error deleting temp archive file: %s", archive.ArchiveFile) + return fmt.Errorf("error deleting temp archive file: %s: %w", archive.ArchiveFile, err) } slog.Debug("deleted temporary archive file", @@ -679,7 +678,7 @@ func DeleteArchiveFile(archive *Archive) error { func CreateOrgArchives(ctx context.Context, now time.Time, config *Config, db *sqlx.DB, s3Client s3iface.S3API, org Org, archiveType ArchiveType) ([]*Archive, []*Archive, []*Archive, []*Archive, error) { archiveCount, err := GetCurrentArchiveCount(ctx, db, org, archiveType) if err != nil { - return nil, nil, nil, nil, errors.Wrapf(err, "error getting current archive count") + return nil, nil, nil, nil, fmt.Errorf("error getting current archive count: %w", err) } var dailiesCreated, dailiesFailed, monthliesCreated, monthliesFailed []*Archive @@ -688,7 +687,7 @@ func CreateOrgArchives(ctx context.Context, now time.Time, config *Config, db *s if archiveCount == 0 { archives, err := GetMissingMonthlyArchives(ctx, db, now, org, archiveType) if err != nil { - return nil, nil, nil, nil, errors.Wrapf(err, "error getting missing monthly archives") + return nil, nil, nil, nil, fmt.Errorf("error getting missing monthly archives: %w", err) } // we first create monthly archives @@ -698,7 +697,7 @@ func CreateOrgArchives(ctx context.Context, now time.Time, config *Config, db *s // then add in daily archives taking into account the monthly that have been built daily, err := GetMissingDailyArchives(ctx, db, now, org, archiveType) if err != nil { - return nil, nil, nil, nil, errors.Wrapf(err, "error getting missing daily archives") + return nil, nil, nil, nil, fmt.Errorf("error getting missing daily archives: %w", err) } // we then create missing daily archives @@ -712,7 +711,7 @@ func CreateOrgArchives(ctx context.Context, now time.Time, config *Config, db *s func createArchive(ctx context.Context, db *sqlx.DB, config *Config, s3Client s3iface.S3API, archive *Archive) error { err := CreateArchiveFile(ctx, db, archive, config.TempDir) if err != nil { - return errors.Wrap(err, "error writing archive file") + return fmt.Errorf("error writing archive file: %w", err) } defer func() { @@ -727,13 +726,13 @@ func createArchive(ctx context.Context, db *sqlx.DB, config *Config, s3Client s3 if config.UploadToS3 { err = UploadArchive(ctx, s3Client, config.S3Bucket, archive) if err != nil { - return errors.Wrap(err, "error writing archive to s3") + return fmt.Errorf("error writing archive to s3: %w", err) } } err = WriteArchiveToDB(ctx, db, archive) if err != nil { - return errors.Wrap(err, "error writing record to db") + return fmt.Errorf("error writing record to db: %w", err) } return nil @@ -883,7 +882,7 @@ func ArchiveOrg(ctx context.Context, now time.Time, cfg *Config, db *sqlx.DB, s3 dailiesCreated, dailiesFailed, monthliesCreated, monthliesFailed, err := CreateOrgArchives(ctx, now, cfg, db, s3Client, org, archiveType) if err != nil { - return nil, nil, nil, nil, nil, errors.Wrapf(err, "error creating archives") + return nil, nil, nil, nil, nil, fmt.Errorf("error creating archives: %w", err) } if len(dailiesCreated) > 0 { @@ -894,7 +893,7 @@ func ArchiveOrg(ctx context.Context, now time.Time, cfg *Config, db *sqlx.DB, s3 rollupsCreated, rollupsFailed, err := RollupOrgArchives(ctx, now, cfg, db, s3Client, org, archiveType) if err != nil { - return nil, nil, nil, nil, nil, errors.Wrapf(err, "error rolling up archives") + return nil, nil, nil, nil, nil, fmt.Errorf("error rolling up archives: %w", err) } monthliesCreated = append(monthliesCreated, rollupsCreated...) @@ -906,7 +905,7 @@ func ArchiveOrg(ctx context.Context, now time.Time, cfg *Config, db *sqlx.DB, s3 if cfg.Delete { deleted, err = DeleteArchivedOrgRecords(ctx, now, cfg, db, s3Client, org, archiveType) if err != nil { - return dailiesCreated, dailiesFailed, monthliesCreated, monthliesFailed, nil, errors.Wrapf(err, "error deleting archived records") + return dailiesCreated, dailiesFailed, monthliesCreated, monthliesFailed, nil, fmt.Errorf("error deleting archived records: %w", err) } } @@ -923,7 +922,7 @@ func ArchiveActiveOrgs(db *sqlx.DB, cfg *Config, s3Client s3iface.S3API) error { cancel() if err != nil { - return errors.Wrap(err, "error getting active orgs") + return fmt.Errorf("error getting active orgs: %w", err) } totalRunsRecordsArchived, totalMsgsRecordsArchived := 0, 0 diff --git a/archives/messages.go b/archives/messages.go index 647b211..f7362ca 100644 --- a/archives/messages.go +++ b/archives/messages.go @@ -10,7 +10,6 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/jmoiron/sqlx" "github.com/nyaruka/gocommon/dates" - "github.com/pkg/errors" ) const ( @@ -75,14 +74,14 @@ func writeMessageRecords(ctx context.Context, db *sqlx.DB, archive *Archive, wri rows, err := db.QueryxContext(ctx, sqlLookupMsgs, archive.Org.ID, archive.StartDate, archive.endDate()) if err != nil { - return 0, errors.Wrapf(err, "error querying messages for org: %d", archive.Org.ID) + return 0, fmt.Errorf("error querying messages for org: %d: %w", archive.Org.ID, err) } defer rows.Close() for rows.Next() { err = rows.Scan(&visibility, &record) if err != nil { - return 0, errors.Wrapf(err, "error scanning message row for org: %d", archive.Org.ID) + return 0, fmt.Errorf("error scanning message row for org: %d: %w", archive.Org.ID, err) } if visibility == "deleted" { @@ -193,19 +192,19 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 // first delete any labelings err = executeInQuery(ctx, tx, sqlDeleteMessageLabels, idBatch) if err != nil { - return errors.Wrap(err, "error removing message labels") + return fmt.Errorf("error removing message labels: %w", err) } // then delete the messages themselves err = executeInQuery(ctx, tx, sqlDeleteMessages, idBatch) if err != nil { - return errors.Wrap(err, "error deleting messages") + return fmt.Errorf("error deleting messages: %w", err) } // commit our transaction err = tx.Commit() if err != nil { - return errors.Wrap(err, "error committing message delete transaction") + return fmt.Errorf("error committing message delete transaction: %w", err) } log.Debug("deleted batch of messages", "elapsed", dates.Since(start), "count", len(idBatch)) @@ -221,7 +220,7 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 // all went well! mark our archive as no longer needing deletion _, err = db.ExecContext(outer, sqlUpdateArchiveDeleted, archive.ID, deletedOn) if err != nil { - return errors.Wrap(err, "error setting archive as deleted") + return fmt.Errorf("error setting archive as deleted: %w", err) } archive.NeedsDeletion = false archive.DeletedOn = &deletedOn @@ -262,46 +261,46 @@ func DeleteBroadcasts(ctx context.Context, now time.Time, config *Config, db *sq var broadcastID int64 if err := rows.Scan(&broadcastID); err != nil { - return errors.Wrap(err, "unable to get broadcast id") + return fmt.Errorf("unable to get broadcast id: %w", err) } // we delete broadcasts in a transaction per broadcast tx, err := db.BeginTx(ctx, nil) if err != nil { - return errors.Wrapf(err, "error starting transaction while deleting broadcast: %d", broadcastID) + return fmt.Errorf("error starting transaction while deleting broadcast: %d: %w", broadcastID, err) } // delete contacts M2M _, err = tx.Exec(`DELETE from msgs_broadcast_contacts WHERE broadcast_id = $1`, broadcastID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting related contacts for broadcast: %d", broadcastID) + return fmt.Errorf("error deleting related contacts for broadcast: %d: %w", broadcastID, err) } // delete groups M2M _, err = tx.Exec(`DELETE from msgs_broadcast_groups WHERE broadcast_id = $1`, broadcastID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting related groups for broadcast: %d", broadcastID) + return fmt.Errorf("error deleting related groups for broadcast: %d: %w", broadcastID, err) } // delete counts associated with this broadcast _, err = tx.Exec(`DELETE from msgs_broadcastmsgcount WHERE broadcast_id = $1`, broadcastID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting counts for broadcast: %d", broadcastID) + return fmt.Errorf("error deleting counts for broadcast: %d: %w", broadcastID, err) } // finally, delete our broadcast _, err = tx.Exec(`DELETE from msgs_broadcast WHERE id = $1`, broadcastID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting broadcast: %d", broadcastID) + return fmt.Errorf("error deleting broadcast: %d: %w", broadcastID, err) } err = tx.Commit() if err != nil { - return errors.Wrapf(err, "error deleting broadcast: %d", broadcastID) + return fmt.Errorf("error deleting broadcast: %d: %w", broadcastID, err) } count++ diff --git a/archives/runs.go b/archives/runs.go index b9ea8e7..d713322 100644 --- a/archives/runs.go +++ b/archives/runs.go @@ -10,7 +10,6 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/jmoiron/sqlx" "github.com/nyaruka/gocommon/dates" - "github.com/pkg/errors" ) const ( @@ -62,7 +61,7 @@ func writeRunRecords(ctx context.Context, db *sqlx.DB, archive *Archive, writer var rows *sqlx.Rows rows, err := db.QueryxContext(ctx, sqlLookupRuns, archive.Org.ID, archive.StartDate, archive.endDate()) if err != nil { - return 0, errors.Wrapf(err, "error querying run records for org: %d", archive.Org.ID) + return 0, fmt.Errorf("error querying run records for org: %d: %w", archive.Org.ID, err) } defer rows.Close() @@ -76,7 +75,7 @@ func writeRunRecords(ctx context.Context, db *sqlx.DB, archive *Archive, writer err = rows.Scan(&runUUID, &runExitedOn, &record) if err != nil { - return 0, errors.Wrapf(err, "error scanning run record for org: %d", archive.Org.ID) + return 0, fmt.Errorf("error scanning run record for org: %d: %w", archive.Org.ID, err) } // shouldn't be archiving an active run, that's an error @@ -188,13 +187,13 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie // delete our runs err = executeInQuery(ctx, tx, sqlDeleteRuns, idBatch) if err != nil { - return errors.Wrap(err, "error deleting runs") + return fmt.Errorf("error deleting runs: %w", err) } // commit our transaction err = tx.Commit() if err != nil { - return errors.Wrap(err, "error committing run delete transaction") + return fmt.Errorf("error committing run delete transaction: %w", err) } log.Debug("deleted batch of runs", "elapsed", dates.Since(start), "count", len(idBatch)) @@ -210,7 +209,7 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie // all went well! mark our archive as no longer needing deletion _, err = db.ExecContext(outer, sqlUpdateArchiveDeleted, archive.ID, deletedOn) if err != nil { - return errors.Wrap(err, "error setting archive as deleted") + return fmt.Errorf("error setting archive as deleted: %w", err) } archive.NeedsDeletion = false archive.DeletedOn = &deletedOn @@ -250,53 +249,53 @@ func DeleteFlowStarts(ctx context.Context, now time.Time, config *Config, db *sq var startID int64 if err := rows.Scan(&startID); err != nil { - return errors.Wrap(err, "unable to get start id") + return fmt.Errorf("unable to get start id: %w", err) } // we delete starts in a transaction per start tx, err := db.BeginTx(ctx, nil) if err != nil { - return errors.Wrapf(err, "error starting transaction while deleting start: %d", startID) + return fmt.Errorf("error starting transaction while deleting start: %d: %w", startID, err) } // delete contacts M2M _, err = tx.Exec(`DELETE from flows_flowstart_contacts WHERE flowstart_id = $1`, startID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting related contacts for start: %d", startID) + return fmt.Errorf("error deleting related contacts for start: %d: %w", startID, err) } // delete groups M2M _, err = tx.Exec(`DELETE from flows_flowstart_groups WHERE flowstart_id = $1`, startID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting related groups for start: %d", startID) + return fmt.Errorf("error deleting related groups for start: %d: %w", startID, err) } // delete calls M2M _, err = tx.Exec(`DELETE from flows_flowstart_calls WHERE flowstart_id = $1`, startID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting related calls for start: %d", startID) + return fmt.Errorf("error deleting related calls for start: %d: %w", startID, err) } // delete counts _, err = tx.Exec(`DELETE from flows_flowstartcount WHERE start_id = $1`, startID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting counts for start: %d", startID) + return fmt.Errorf("error deleting counts for start: %d: %w", startID, err) } // finally, delete our start _, err = tx.Exec(`DELETE from flows_flowstart WHERE id = $1`, startID) if err != nil { tx.Rollback() - return errors.Wrapf(err, "error deleting start: %d", startID) + return fmt.Errorf("error deleting start: %d: %w", startID, err) } err = tx.Commit() if err != nil { - return errors.Wrapf(err, "error deleting start: %d", startID) + return fmt.Errorf("error deleting start: %d: %w", startID, err) } count++ diff --git a/go.mod b/go.mod index e3002cb..8e3c60c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.3.0 github.com/nyaruka/gocommon v1.53.2 - github.com/pkg/errors v0.9.1 github.com/samber/slog-multi v1.0.2 github.com/samber/slog-sentry v1.2.2 github.com/stretchr/testify v1.9.0 @@ -25,6 +24,7 @@ require ( github.com/nyaruka/librato v1.1.1 // indirect github.com/nyaruka/null/v2 v2.0.3 // indirect github.com/nyaruka/phonenumbers v1.3.4 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/samber/lo v1.38.1 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect From f5389f1c71be75f9071d5f1cea27afa49859e1b6 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 3 Jun 2024 13:29:56 -0500 Subject: [PATCH 11/18] Update deps --- go.mod | 25 +++++++++++----------- go.sum | 65 ++++++++++++++++++++++++++++++++++------------------------ 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 8e3c60c..2c07640 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/nyaruka/rp-archiver go 1.22 require ( - github.com/aws/aws-sdk-go v1.51.20 - github.com/getsentry/sentry-go v0.22.0 - github.com/jmoiron/sqlx v1.3.5 + github.com/aws/aws-sdk-go v1.53.14 + github.com/getsentry/sentry-go v0.28.0 + github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.3.0 - github.com/nyaruka/gocommon v1.53.2 - github.com/samber/slog-multi v1.0.2 + github.com/nyaruka/gocommon v1.55.5 + github.com/samber/slog-multi v1.0.3 github.com/samber/slog-sentry v1.2.2 github.com/stretchr/testify v1.9.0 ) @@ -17,19 +17,20 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/structs v1.1.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect github.com/nyaruka/librato v1.1.1 // indirect github.com/nyaruka/null/v2 v2.0.3 // indirect - github.com/nyaruka/phonenumbers v1.3.4 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/nyaruka/phonenumbers v1.3.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/samber/lo v1.38.1 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + github.com/samber/lo v1.39.0 // indirect + golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 51f1390..97e86c7 100644 --- a/go.sum +++ b/go.sum @@ -1,45 +1,50 @@ -github.com/aws/aws-sdk-go v1.51.20 h1:ziM90ujYHKKkoTZL+Wg2LwjbQecL+l298GGJeG4ktZs= -github.com/aws/aws-sdk-go v1.51.20/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/aws/aws-sdk-go v1.53.14 h1:SzhkC2Pzag0iRW8WBb80RzKdGXDydJR9LAMs2GyKJ2M= +github.com/aws/aws-sdk-go v1.53.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluWaP6dM= -github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.28.0 h1:7Rqx9M3ythTKy2J6uZLHmc8Sz9OGgIlseuO1iBX/s0M= +github.com/getsentry/sentry-go v0.28.0/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.3.0 h1:kGvJqVN8AHowb4HdaHAviJ0Z3yI5Pyekp1WqibFEaGk= github.com/nyaruka/ezconf v0.3.0/go.mod h1:89GUW6EPRNLIxT7lC4LWnjWTgZeQwRoX7lBmc8ralAU= -github.com/nyaruka/gocommon v1.53.2 h1:DKXlvNUcim/+4X0Dz8BjdDHmQBNVU60giEOVu2faRCE= -github.com/nyaruka/gocommon v1.53.2/go.mod h1:1E8KKcJ4r+FPWQm5ImGjiH6K96dKInjLM9BkcD4zOUQ= +github.com/nyaruka/gocommon v1.55.5 h1:1HCDTwoegsmd1FJQH/cFPHIzX3YEtqjRTJ09N+4nlSY= +github.com/nyaruka/gocommon v1.55.5/go.mod h1:hHczEMFfODl6k527y3yxQv48WDX5BnBT0/WpCveSYwg= github.com/nyaruka/librato v1.1.1 h1:0nTYtJLl3Sn7lX3CuHsLf+nXy1k/tGV0OjVxLy3Et4s= github.com/nyaruka/librato v1.1.1/go.mod h1:fme1Fu1PT2qvkaBZyw8WW+SrnFe2qeeCWpvqmAaKAKE= github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw= github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA= -github.com/nyaruka/phonenumbers v1.3.4 h1:bF1Wdh++fxw09s3surhVeBhXEcUKG07pHeP8HQXqjn8= -github.com/nyaruka/phonenumbers v1.3.4/go.mod h1:Ut+eFwikULbmCenH6InMKL9csUNLyxHuBLyfkpum11s= +github.com/nyaruka/phonenumbers v1.3.6 h1:33owXWp4d1U+Tyaj9fpci6PbvaQZcXBUO2FybeKeLwQ= +github.com/nyaruka/phonenumbers v1.3.6/go.mod h1:Ut+eFwikULbmCenH6InMKL9csUNLyxHuBLyfkpum11s= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -48,23 +53,29 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/samber/slog-multi v1.0.2 h1:6BVH9uHGAsiGkbbtQgAOQJMpKgV8unMrHhhJaw+X1EQ= github.com/samber/slog-multi v1.0.2/go.mod h1:uLAvHpGqbYgX4FSL0p1ZwoLuveIAJvBECtE07XmYvFo= +github.com/samber/slog-multi v1.0.3 h1:8wlX8ioZE38h91DwoJBVnC7JfhgwERwlekY+NHsVsv0= +github.com/samber/slog-multi v1.0.3/go.mod h1:TvwgIK4XPBb8Dn18as5uiTHf7in8gN/AtUXsT57UYuo= github.com/samber/slog-sentry v1.2.2 h1:S0glIVITlGCCfSvIOte2Sh63HMHJpYN3hDr+97hILIk= github.com/samber/slog-sentry v1.2.2/go.mod h1:bHm8jm1dks0p+xc/lH2i4TIFwnPcMTvZeHgCBj5+uhA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d h1:N0hmiNbwsSNwHBAvR3QB5w25pUwH4tK0Y/RltD1j1h4= +golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 003d1f8a255244f272fd1eddd023b94fd7ac4912 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 4 Jun 2024 08:15:34 -0500 Subject: [PATCH 12/18] Update slog-multi --- go.mod | 6 ++---- go.sum | 18 ++++-------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 2c07640..e8b3b65 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/nyaruka/rp-archiver go 1.22 require ( - github.com/aws/aws-sdk-go v1.53.14 + github.com/aws/aws-sdk-go v1.53.15 github.com/getsentry/sentry-go v0.28.0 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.3.0 github.com/nyaruka/gocommon v1.55.5 - github.com/samber/slog-multi v1.0.3 + github.com/samber/slog-multi v1.1.0 github.com/samber/slog-sentry v1.2.2 github.com/stretchr/testify v1.9.0 ) @@ -17,8 +17,6 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/structs v1.1.0 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect diff --git a/go.sum b/go.sum index 97e86c7..2eae22a 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/aws/aws-sdk-go v1.53.14 h1:SzhkC2Pzag0iRW8WBb80RzKdGXDydJR9LAMs2GyKJ2M= -github.com/aws/aws-sdk-go v1.53.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.53.15 h1:FtZmkg7xM8RfP2oY6p7xdKBYrRgkITk9yve2QV7N938= +github.com/aws/aws-sdk-go v1.53.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -15,10 +15,6 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -51,21 +47,15 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= -github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= -github.com/samber/slog-multi v1.0.2 h1:6BVH9uHGAsiGkbbtQgAOQJMpKgV8unMrHhhJaw+X1EQ= -github.com/samber/slog-multi v1.0.2/go.mod h1:uLAvHpGqbYgX4FSL0p1ZwoLuveIAJvBECtE07XmYvFo= -github.com/samber/slog-multi v1.0.3 h1:8wlX8ioZE38h91DwoJBVnC7JfhgwERwlekY+NHsVsv0= -github.com/samber/slog-multi v1.0.3/go.mod h1:TvwgIK4XPBb8Dn18as5uiTHf7in8gN/AtUXsT57UYuo= +github.com/samber/slog-multi v1.1.0 h1:m5wfpXE8Qu2gCiR/JnhFGsLcWDOmTxnso32EMffVAY0= +github.com/samber/slog-multi v1.1.0/go.mod h1:uLAvHpGqbYgX4FSL0p1ZwoLuveIAJvBECtE07XmYvFo= github.com/samber/slog-sentry v1.2.2 h1:S0glIVITlGCCfSvIOte2Sh63HMHJpYN3hDr+97hILIk= github.com/samber/slog-sentry v1.2.2/go.mod h1:bHm8jm1dks0p+xc/lH2i4TIFwnPcMTvZeHgCBj5+uhA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d h1:N0hmiNbwsSNwHBAvR3QB5w25pUwH4tK0Y/RltD1j1h4= -golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= From 22d74e2922aa35c4ea0560e891b9977c733966c7 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 4 Jun 2024 14:27:53 -0500 Subject: [PATCH 13/18] Update CHANGELOG.md for v9.1.2 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1ab34..7dc2f57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v9.1.2 (2024-06-04) +------------------------- + * Update deps + * Use std lib errors + v9.1.1 (2024-04-25) ------------------------- * Add support for status=READ From 5c7a842800a028bb235a7eb4752530cbe2b2e6c5 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 5 Jun 2024 14:27:20 -0500 Subject: [PATCH 14/18] Update goreleaser config to v2 --- .github/workflows/ci.yml | 2 +- .goreleaser.yaml | 20 ++++++++++++++++++++ goreleaser.yml | 19 ------------------- 3 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 .goreleaser.yaml delete mode 100644 goreleaser.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7236bc7..a3bff9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: go-version: ${{ env.go-version }} - name: Publish release - uses: goreleaser/goreleaser-action@v5 + uses: goreleaser/goreleaser-action@v6 with: args: release --clean env: diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..ee6852a --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,20 @@ +version: 2 +builds: + - main: ./cmd/rp-archiver/main.go + binary: rp-archiver + goos: + - darwin + - linux + goarch: + - amd64 + - arm64 + +changelog: + filters: + exclude: + - "^Update CHANGELOG.md" + +archives: + - files: + - LICENSE + - README.md diff --git a/goreleaser.yml b/goreleaser.yml deleted file mode 100644 index a5023d4..0000000 --- a/goreleaser.yml +++ /dev/null @@ -1,19 +0,0 @@ -build: - main: ./cmd/rp-archiver/main.go - binary: rp-archiver - goos: - - darwin - - linux - goarch: - - amd64 - - arm64 - -changelog: - filters: - exclude: - - "^Update CHANGELOG.md" - -archives: - - files: - - LICENSE - - README.md From 59323de1681dab6fc42bd93b85a82f2347feeab0 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 5 Jun 2024 15:39:37 -0500 Subject: [PATCH 15/18] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a9339e8..8083ff0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Archiver +# 🗜️ Archiver [![Build Status](https://github.com/nyaruka/rp-archiver/workflows/CI/badge.svg)](https://github.com/nyaruka/rp-archiver/actions?query=workflow%3ACI) [![codecov](https://codecov.io/gh/nyaruka/rp-archiver/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/rp-archiver) From 5d57272963daf091776f5648f04b0a901f20ec46 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 17 Jul 2024 15:48:30 -0500 Subject: [PATCH 16/18] Test against postgresql 15 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3bff9e..da3d302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: services: postgres: - image: postgis/postgis:14-3.3-alpine + image: postgres:15-alpine env: POSTGRES_DB: archiver_test POSTGRES_USER: archiver_test From dcbf3f871c2ca006fd544283ad3107a5faa944e4 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 17 Jul 2024 15:53:20 -0500 Subject: [PATCH 17/18] Update dependencies --- go.mod | 20 ++++++++++---------- go.sum | 42 ++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index e8b3b65..6e95b26 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/nyaruka/rp-archiver go 1.22 require ( - github.com/aws/aws-sdk-go v1.53.15 - github.com/getsentry/sentry-go v0.28.0 + github.com/aws/aws-sdk-go v1.54.19 + github.com/getsentry/sentry-go v0.28.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.3.0 - github.com/nyaruka/gocommon v1.55.5 - github.com/samber/slog-multi v1.1.0 + github.com/nyaruka/gocommon v1.55.8 + github.com/samber/slog-multi v1.2.0 github.com/samber/slog-sentry v1.2.2 github.com/stretchr/testify v1.9.0 ) @@ -23,12 +23,12 @@ require ( github.com/naoina/toml v0.1.1 // indirect github.com/nyaruka/librato v1.1.1 // indirect github.com/nyaruka/null/v2 v2.0.3 // indirect - github.com/nyaruka/phonenumbers v1.3.6 // indirect + github.com/nyaruka/phonenumbers v1.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/samber/lo v1.39.0 // indirect - golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + github.com/samber/lo v1.46.0 // indirect + golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2eae22a..1a6fd4b 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/aws/aws-sdk-go v1.53.15 h1:FtZmkg7xM8RfP2oY6p7xdKBYrRgkITk9yve2QV7N938= -github.com/aws/aws-sdk-go v1.53.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= +github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/getsentry/sentry-go v0.28.0 h1:7Rqx9M3ythTKy2J6uZLHmc8Sz9OGgIlseuO1iBX/s0M= -github.com/getsentry/sentry-go v0.28.0/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= +github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= +github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= @@ -33,39 +33,37 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.3.0 h1:kGvJqVN8AHowb4HdaHAviJ0Z3yI5Pyekp1WqibFEaGk= github.com/nyaruka/ezconf v0.3.0/go.mod h1:89GUW6EPRNLIxT7lC4LWnjWTgZeQwRoX7lBmc8ralAU= -github.com/nyaruka/gocommon v1.55.5 h1:1HCDTwoegsmd1FJQH/cFPHIzX3YEtqjRTJ09N+4nlSY= -github.com/nyaruka/gocommon v1.55.5/go.mod h1:hHczEMFfODl6k527y3yxQv48WDX5BnBT0/WpCveSYwg= +github.com/nyaruka/gocommon v1.55.8 h1:vMMnwCt/P5D7kWic13g3TkALp6uTpR/a9JIW22DITh0= +github.com/nyaruka/gocommon v1.55.8/go.mod h1:ZiVNGrpzkv8str/Tjblddl2tmR0NCifv1mAvjTOKXQI= github.com/nyaruka/librato v1.1.1 h1:0nTYtJLl3Sn7lX3CuHsLf+nXy1k/tGV0OjVxLy3Et4s= github.com/nyaruka/librato v1.1.1/go.mod h1:fme1Fu1PT2qvkaBZyw8WW+SrnFe2qeeCWpvqmAaKAKE= github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw= github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA= -github.com/nyaruka/phonenumbers v1.3.6 h1:33owXWp4d1U+Tyaj9fpci6PbvaQZcXBUO2FybeKeLwQ= -github.com/nyaruka/phonenumbers v1.3.6/go.mod h1:Ut+eFwikULbmCenH6InMKL9csUNLyxHuBLyfkpum11s= +github.com/nyaruka/phonenumbers v1.4.0 h1:ddhWiHnHCIX3n6ETDA58Zq5dkxkjlvgrDWM2OHHPCzU= +github.com/nyaruka/phonenumbers v1.4.0/go.mod h1:gv+CtldaFz+G3vHHnasBSirAi3O2XLqZzVWz4V1pl2E= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= -github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= -github.com/samber/slog-multi v1.1.0 h1:m5wfpXE8Qu2gCiR/JnhFGsLcWDOmTxnso32EMffVAY0= -github.com/samber/slog-multi v1.1.0/go.mod h1:uLAvHpGqbYgX4FSL0p1ZwoLuveIAJvBECtE07XmYvFo= +github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ= +github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/samber/slog-multi v1.2.0 h1:JIebVdmeGkCMd5/ticlmU+aDYl4tdAZBmp5uLaSzr0k= +github.com/samber/slog-multi v1.2.0/go.mod h1:uLAvHpGqbYgX4FSL0p1ZwoLuveIAJvBECtE07XmYvFo= github.com/samber/slog-sentry v1.2.2 h1:S0glIVITlGCCfSvIOte2Sh63HMHJpYN3hDr+97hILIk= github.com/samber/slog-sentry v1.2.2/go.mod h1:bHm8jm1dks0p+xc/lH2i4TIFwnPcMTvZeHgCBj5+uhA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= -golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 h1:wDLEX9a7YQoKdKNQt88rtydkqDxeGaBUTnIYc3iG/mA= +golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 7eea020c7c2d6acb8eacfe6e777415d262470941 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 17 Jul 2024 15:58:51 -0500 Subject: [PATCH 18/18] Update CHANGELOG.md for v9.2.0 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dc2f57..d31fc48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v9.2.0 (2024-07-17) +------------------------- + * Update dependencies + * Test against PostgreSQL 15 + v9.1.2 (2024-06-04) ------------------------- * Update deps