diff --git a/.github/workflows/build-rp-archiver-push-tag-india-ire.yaml b/.github/workflows/build-rp-archiver-push-tag-india-ire.yaml index e39e26f..6728a2f 100644 --- a/.github/workflows/build-rp-archiver-push-tag-india-ire.yaml +++ b/.github/workflows/build-rp-archiver-push-tag-india-ire.yaml @@ -16,7 +16,6 @@ jobs: if grep -qs -e '^.*.*-develop' <<< "${TAG}" ; then echo "Found environment: DEVELOP - ${TAG}" echo "ENVIRONMENT=develop" | tee -a "${GITHUB_ENV}" - exit 1 # stop action elif grep -qs -e '^.*.*-staging' <<< "${TAG}" ; then echo "Found environment: STAGING - ${TAG}" echo "ENVIRONMENT=staging" | tee -a "${GITHUB_ENV}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aebda51..4ad1418 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: "1.19.x" + go-version: "1.21.x" jobs: test: name: Test diff --git a/CHANGELOG.md b/CHANGELOG.md index 3107aad..ce449a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,29 @@ +v9.0.0 (2024-01-05) +------------------------- + * Update dependencies + +v8.3.4 (2023-11-08) +------------------------- + * Fix deleting of broadcasts so we don't include deleted scheduled broadcasts + +v8.3.3 (2023-09-26) +------------------------- + * Allow disabling of hash checking + * Fix checking S3 uploads so that we always check size but only check hash for files uploaded as single part + +v8.3.2 (2023-09-25) +------------------------- + * Update to go 1.21 + +v8.3.1 (2023-09-19) +------------------------- + * Add support for optin type messages + * Update deps and go version for CI + +v8.3.0 (2023-08-10) +------------------------- + * Update to go 1.20 + v8.2.0 (2023-07-31) ------------------------- * Update .gitignore diff --git a/README.md b/README.md index d984471..a9339e8 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,14 @@ We recommend running it with no changes to the configuration and no parameters, environment variables to configure it. You can use `% rp-archiver --help` to see a list of the environment variables and parameters and for more details on each option. -### RapidPro - -For use with RapidPro, you will want to configure these settings: +For use with RapidPro/TextIt, you will need to configure these settings: * `ARCHIVER_DB`: URL describing how to connect to the database (default "postgres://temba:temba@localhost/temba?sslmode=disable") * `ARCHIVER_TEMP_DIR`: The directory that temporary archives will be written before upload (default "/tmp") * `ARCHIVER_DELETE`: Whether to delete messages and runs after they are archived, we recommend setting this to true for large installations (default false) -For writing of archives, Archiver needs access to an S3 bucket, you can configure access to your bucket via: +For writing of archives, Archiver needs access to a storage bucket on an S3 compatible service. For AWS we recommend that +you choose SSE-S3 encryption as this is the only type that supports validation of upload ETags. * `ARCHIVER_S3_REGION`: The region for your S3 bucket (ex: `ew-west-1`) * `ARCHIVER_S3_BUCKET`: The name of your S3 bucket (ex: `dl-archiver-test"`) @@ -42,62 +41,13 @@ For writing of archives, Archiver needs access to an S3 bucket, you can configur * `ARCHIVER_AWS_ACCESS_KEY_ID`: The AWS access key id used to authenticate to AWS * `ARCHIVER_AWS_SECRET_ACCESS_KEY` The AWS secret access key used to authenticate to AWS -Recommended settings for error reporting: - - * `ARCHIVER_SENTRY_DSN`: The DSN to use when logging errors to Sentry +If using a different encryption type or service that produces non-MD5 ETags: -### Reference + * `CHECK_S3_HASHES`: can be set to `FALSE` to disable checking of upload hashes. -These are the configuration options that can be provided as parameters or environment variables. If using environment -varibles, convert to uppercase, replace dashes with underscores and prefix the name with `ARCHIVER_`, e.g. `-log-level` -becomes `ARCHIVER_LOG_LEVEL`. +Recommended settings for error reporting: -``` - -archive-messages - whether we should archive messages (default true) - -archive-runs - whether we should archive runs (default true) - -aws-access-key-id string - the access key id to use when authenticating S3 (default none) - -aws-secret-access-key string - the secret access key id to use when authenticating S3 (default none) - -db string - the connection string for our database (default "postgres://localhost/archiver_test?sslmode=disable") - -debug-conf - print where config values are coming from - -delete - whether to delete messages and runs from the db after archival (default false) - -help - print usage information - -keep-files - whether we should keep local archive files after upload (default false) - -librato-username - the Librato username for metrics reporting - -librato-token - the Librato token for metrics reporting - -log-level string - the log level, one of error, warn, info, debug (default "info") - -once - run archving immediately and then exit - -retention-period int - the number of days to keep before archiving (default 90) - -s3-bucket string - the S3 bucket we will write archives to (default "dl-archiver-test") - -s3-disable-ssl - whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network - -s3-endpoint string - the S3 endpoint we will write archives to (default "https://s3.amazonaws.com") - -s3-force-path-style - whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service - -s3-region string - the S3 region we will write archives to (default "us-east-1") - -sentry-dsn string - the sentry configuration to log errors to, if any - -temp-dir string - directory where temporary archive files are written (default "/tmp") - -upload-to-s3 - whether we should upload archive to S3 (default true) -``` + * `ARCHIVER_SENTRY_DSN`: The DSN to use when logging errors to Sentry ## Development diff --git a/WENI-CHANGELOG.md b/WENI-CHANGELOG.md index 2a7fcf2..b295053 100644 --- a/WENI-CHANGELOG.md +++ b/WENI-CHANGELOG.md @@ -1,3 +1,7 @@ +1.2.4-archiver-9.0.0 +---------- + * update to rp-arciver v9.0.0 + 1.2.4-archiver-8.2.0 ---------- * Merge rp-archiver v8.2.0 diff --git a/archives/archives.go b/archives/archives.go index e906dc4..a6c60cd 100644 --- a/archives/archives.go +++ b/archives/archives.go @@ -535,10 +535,6 @@ func CreateArchiveFile(ctx context.Context, db *sqlx.DB, archive *Archive, archi return errors.Wrapf(err, "error calculating archive hash") } - if stat.Size() > 5e9 { - return fmt.Errorf("archive too large, must be smaller than 5 gigs, build dailies if possible") - } - archive.ArchiveFile = file.Name() archive.Size = stat.Size() archive.RecordCount = recordCount diff --git a/archives/config.go b/archives/config.go index e51fc44..ece442a 100644 --- a/archives/config.go +++ b/archives/config.go @@ -17,9 +17,10 @@ type Config struct { AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` - TempDir string `help:"directory where temporary archive files are written"` - KeepFiles bool `help:"whether we should keep local archive files after upload (default false)"` - UploadToS3 bool `help:"whether we should upload archive to S3"` + TempDir string `help:"directory where temporary archive files are written"` + KeepFiles bool `help:"whether we should keep local archive files after upload (default false)"` + UploadToS3 bool `help:"whether we should upload archive to S3"` + CheckS3Hashes bool `help:"whether to check S3 hashes of uploaded archives before deleting records"` ArchiveMessages bool `help:"whether we should archive messages"` ArchiveRuns bool `help:"whether we should archive runs"` @@ -53,9 +54,10 @@ func NewDefaultConfig() *Config { AWSAccessKeyID: "", AWSSecretAccessKey: "", - TempDir: "/tmp", - KeepFiles: false, - UploadToS3: true, + TempDir: "/tmp", + KeepFiles: false, + UploadToS3: true, + CheckS3Hashes: true, ArchiveMessages: true, ArchiveRuns: true, diff --git a/archives/messages.go b/archives/messages.go index e987b97..f59053a 100644 --- a/archives/messages.go +++ b/archives/messages.go @@ -28,7 +28,12 @@ SELECT rec.visibility, row_to_json(rec) FROM ( row_to_json(channel) as channel, row_to_json(flow) as flow, CASE WHEN direction = 'I' THEN 'in' WHEN direction = 'O' THEN 'out' ELSE NULL END AS direction, - CASE WHEN msg_type = 'V' THEN 'voice' ELSE 'text' END AS "type", + CASE + WHEN msg_type = 'T' THEN 'text' + WHEN msg_type = 'O' THEN 'optin' + WHEN msg_type = 'V' THEN 'voice' + ELSE NULL + END AS "type", CASE WHEN status = 'I' THEN 'initializing' WHEN status = 'P' THEN 'queued' @@ -124,15 +129,19 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 }) log.Info("deleting messages") - // first things first, make sure our file is present on S3 - md5, err := GetS3FileETAG(outer, s3Client, archive.URL) + // first things first, make sure our file is correct on S3 + s3Size, s3Hash, err := GetS3FileInfo(outer, s3Client, archive.URL) if err != nil { return err } - // if our etag and archive md5 don't match, that's an error, return - if md5 != archive.Hash { - return fmt.Errorf("archive md5: %s and s3 etag: %s do not match", archive.Hash, md5) + if s3Size != archive.Size { + return fmt.Errorf("archive size: %d and s3 size: %d do not match", archive.Size, s3Size) + } + + // if S3 hash is MD5 then check against archive hash + if config.CheckS3Hashes && archive.Size <= maxSingleUploadBytes && s3Hash != archive.Hash { + return fmt.Errorf("archive md5: %s and s3 etag: %s do not match", archive.Hash, s3Hash) } // ok, archive file looks good, let's build up our list of message ids, this may be big but we are int64s so shouldn't be too big @@ -225,7 +234,7 @@ func DeleteArchivedMessages(ctx context.Context, config *Config, db *sqlx.DB, s3 const sqlSelectOldOrgBroadcasts = ` SELECT id FROM msgs_broadcast b - WHERE b.org_id = $1 AND b.created_on < $2 AND b.schedule_id IS NULL AND NOT EXISTS (SELECT 1 FROM msgs_msg WHERE broadcast_id = b.id) + WHERE b.org_id = $1 AND b.created_on < $2 AND b.schedule_id IS NULL AND b.is_active AND NOT EXISTS (SELECT 1 FROM msgs_msg WHERE broadcast_id = b.id) LIMIT 1000000;` // DeleteBroadcasts deletes all broadcasts older than 90 days for the passed in org which have no associated messages diff --git a/archives/runs.go b/archives/runs.go index 87eabfc..9a35957 100644 --- a/archives/runs.go +++ b/archives/runs.go @@ -124,15 +124,19 @@ func DeleteArchivedRuns(ctx context.Context, config *Config, db *sqlx.DB, s3Clie }) log.Info("deleting runs") - // first things first, make sure our file is present on S3 - md5, err := GetS3FileETAG(outer, s3Client, archive.URL) + // first things first, make sure our file is correct on S3 + s3Size, s3Hash, err := GetS3FileInfo(outer, s3Client, archive.URL) if err != nil { return err } - // if our etag and archive md5 don't match, that's an error, return - if md5 != archive.Hash { - return fmt.Errorf("archive md5: %s and s3 etag: %s do not match", archive.Hash, md5) + if s3Size != archive.Size { + return fmt.Errorf("archive size: %d and s3 size: %d do not match", archive.Size, s3Size) + } + + // if S3 hash is MD5 then check against archive hash + if config.CheckS3Hashes && archive.Size <= maxSingleUploadBytes && s3Hash != archive.Hash { + return fmt.Errorf("archive md5: %s and s3 etag: %s do not match", archive.Hash, s3Hash) } // ok, archive file looks good, let's build up our list of run ids, this may be big but we are int64s so shouldn't be too big diff --git a/archives/s3.go b/archives/s3.go index cc55d65..5e7c3ca 100644 --- a/archives/s3.go +++ b/archives/s3.go @@ -20,7 +20,13 @@ import ( "github.com/sirupsen/logrus" ) -var s3BucketURL = "https://%s.s3.amazonaws.com%s" +const s3BucketURL = "https://%s.s3.amazonaws.com%s" + +// any file over this needs to be uploaded in chunks +const maxSingleUploadBytes = 5e9 // 5GB + +// size of chunk to use when doing multi-part uploads +const chunkSizeBytes = 1e9 // 1GB // NewS3Client creates a new s3 client from the passed in config, testing it as necessary func NewS3Client(config *Config) (s3iface.S3API, error) { @@ -81,7 +87,7 @@ func UploadToS3(ctx context.Context, s3Client s3iface.S3API, bucket string, path md5 := base64.StdEncoding.EncodeToString(hashBytes) // if this fits into a single part, upload that way - if archive.Size <= 5e9 { + if archive.Size <= maxSingleUploadBytes { params := &s3.PutObjectInput{ Bucket: aws.String(bucket), Body: f, @@ -97,11 +103,11 @@ func UploadToS3(ctx context.Context, s3Client s3iface.S3API, bucket string, path return err } } else { - // this file is bigger than 5 gigs, use an upload manager instead, it will take care of uploading in parts + // this file is bigger than limit, use an upload manager instead, it will take care of uploading in parts uploader := s3manager.NewUploaderWithClient( s3Client, func(u *s3manager.Uploader) { - u.PartSize = 1e9 // 1 gig per part + u.PartSize = chunkSizeBytes }, ) params := &s3manager.UploadInput{ @@ -129,17 +135,17 @@ func withAcceptEncoding(e string) request.Option { } } -// GetS3FileETAG returns the ETAG hash for the passed in file -func GetS3FileETAG(ctx context.Context, s3Client s3iface.S3API, fileURL string) (string, error) { +// GetS3FileInfo returns the ETAG hash for the passed in file +func GetS3FileInfo(ctx context.Context, s3Client s3iface.S3API, fileURL string) (int64, string, error) { u, err := url.Parse(fileURL) if err != nil { - return "", err + return 0, "", err } bucket := strings.Split(u.Host, ".")[0] path := u.Path - output, err := s3Client.HeadObjectWithContext( + head, err := s3Client.HeadObjectWithContext( ctx, &s3.HeadObjectInput{ Bucket: aws.String(bucket), @@ -148,16 +154,17 @@ func GetS3FileETAG(ctx context.Context, s3Client s3iface.S3API, fileURL string) ) if err != nil { - return "", err + return 0, "", err } - if output.ETag == nil { - return "", fmt.Errorf("no ETAG for object") + if head.ContentLength == nil || head.ETag == nil { + return 0, "", fmt.Errorf("no size or ETag returned for S3 object") } // etag is quoted, remove them - etag := strings.Trim(*output.ETag, `"`) - return etag, nil + etag := strings.Trim(*head.ETag, `"`) + + return *head.ContentLength, etag, nil } // GetS3File return an io.ReadCloser for the passed in bucket and path diff --git a/docker/Dockerfile b/docker/Dockerfile index c4f1fb2..cd5c4c9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,21 +1,19 @@ -FROM golang:1.19-alpine3.18 AS builder +FROM golang:1.23-bookworm AS builder -WORKDIR /app - -RUN apk update \ - && apk add --virtual build-deps gcc git \ - && rm -rf /var/cache/apk/* +WORKDIR /src -COPY . . +COPY go.mod go.sum ./ +RUN go mod download -x -RUN --mount=type=cache,target=/go/pkg/mod/ \ - go install -v ./cmd/... +COPY . ./ -FROM alpine:3.18 +RUN GOOS=linux GOARCH=amd64 go build -o /bin/rp-archiver ./cmd/rp-archiver/*.go -COPY --from=builder /go/bin/ /app/ +FROM gcr.io/distroless/base-debian12 WORKDIR /app +COPY --from=builder bin/rp-archiver ./ + EXPOSE 8080 ENTRYPOINT ["./rp-archiver"] diff --git a/go.mod b/go.mod index 0f5909d..dc2b97d 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module github.com/nyaruka/rp-archiver -go 1.19 +go 1.21 require ( - github.com/aws/aws-sdk-go v1.44.217 + github.com/aws/aws-sdk-go v1.49.15 github.com/evalphobia/logrus_sentry v0.8.2 github.com/jmoiron/sqlx v1.3.5 - github.com/lib/pq v1.10.7 + github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.35.0 + github.com/nyaruka/gocommon v1.42.7 github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.9.0 - github.com/stretchr/testify v1.8.2 + github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.4 ) require ( @@ -23,8 +23,13 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect - github.com/nyaruka/librato v1.0.0 // indirect + github.com/nyaruka/librato v1.1.1 // indirect + github.com/nyaruka/null/v2 v2.0.3 // indirect + github.com/nyaruka/phonenumbers v1.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.6.0 // indirect + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5d77f45..7b83166 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/aws/aws-sdk-go v1.44.217 h1:FcWC56MRl+k756aH3qeMQTylSdeJ58WN0iFz3fkyRz0= -github.com/aws/aws-sdk-go v1.44.217/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k= +github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -14,18 +14,19 @@ github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JY github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= @@ -34,64 +35,37 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.35.0 h1:ICr1kVVBEK0chVmw2yiLwrSGSmRWWFLlX1JhoEpgdp8= -github.com/nyaruka/gocommon v1.35.0/go.mod h1:HaUQmWPrZfKS9MLnXKQj28zF4KlJrzFou+DGuqT7RbE= -github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= -github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= +github.com/nyaruka/gocommon v1.42.7 h1:4U7Ta1LIHVc/uv8sfqmmV5oRiFU8TcJM9a7QjxVoaeA= +github.com/nyaruka/gocommon v1.42.7/go.mod h1:DMj0TJPT2zi6eoXrBSsJTGBxSAUkpBk+UzcMyAbq5DA= +github.com/nyaruka/librato v1.1.1 h1:0nTYtJLl3Sn7lX3CuHsLf+nXy1k/tGV0OjVxLy3Et4s= +github.com/nyaruka/librato v1.1.1/go.mod h1:fme1Fu1PT2qvkaBZyw8WW+SrnFe2qeeCWpvqmAaKAKE= +github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw= +github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA= +github.com/nyaruka/phonenumbers v1.3.0 h1:IFyyJfF2Elg8xGKFghWrRXzb6qAHk+Q3uPqmIgS20JQ= +github.com/nyaruka/phonenumbers v1.3.0/go.mod h1:4jyKp/BFUokLbCHyoZag+T3S1KezFVoEKtgnbpzItC4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/testdb.sql b/testdb.sql index 8504520..892aa83 100644 --- a/testdb.sql +++ b/testdb.sql @@ -59,9 +59,9 @@ CREATE TABLE contacts_contacturn ( priority integer NOT NULL, path character varying(255) NOT NULL, channel_id integer, - auth text, display character varying(255), - identity character varying(255) NOT NULL + identity character varying(255) NOT NULL, + auth_tokens jsonb ); CREATE TABLE contacts_contactgroup ( @@ -116,7 +116,8 @@ CREATE TABLE msgs_broadcast ( org_id integer NOT NULL REFERENCES orgs_org(id), translations jsonb NOT NULL, created_on timestamp with time zone NOT NULL, - schedule_id int NULL + schedule_id int NULL, + is_active boolean NOT NULL ); CREATE TABLE msgs_broadcast_contacts ( @@ -276,21 +277,21 @@ INSERT INTO flows_flow(id, uuid, org_id, name) VALUES (3, '3914b88e-625b-4603-bd9f-9319dc331c6b', 2, 'Flow 3'), (4, 'cfa2371d-2f06-481d-84b2-d974f3803bb0', 2, 'Flow 4'); -INSERT INTO msgs_broadcast(id, org_id, translations, created_on, schedule_id) VALUES -(1, 2, '{"text": {"eng": "hello", "fre": "bonjour"}}', '2017-08-12 22:11:59.890662+02:00', 1), -(2, 2, '{"text": {"und": "hola"}}', '2017-08-12 22:11:59.890662+02:00', NULL), -(3, 2, '{"text": {"und": "not purged"}}', '2017-08-12 19:11:59.890662+02:00', NULL), -(4, 2, '{"text": {"und": "new"}}', '2019-08-12 19:11:59.890662+02:00', NULL); +INSERT INTO msgs_broadcast(id, org_id, translations, created_on, schedule_id, is_active) VALUES +(1, 2, '{"text": {"eng": "hello", "fre": "bonjour"}}', '2017-08-12 22:11:59.890662+02:00', 1, TRUE), +(2, 2, '{"text": {"und": "hola"}}', '2017-08-12 22:11:59.890662+02:00', NULL, TRUE), +(3, 2, '{"text": {"und": "not purged"}}', '2017-08-12 19:11:59.890662+02:00', NULL, TRUE), +(4, 2, '{"text": {"und": "new"}}', '2019-08-12 19:11:59.890662+02:00', NULL, TRUE); INSERT INTO msgs_msg(id, uuid, org_id, broadcast_id, text, created_on, sent_on, modified_on, direction, status, visibility, msg_type, attachments, channel_id, contact_id, contact_urn_id, flow_id, msg_count, error_count, next_attempt) VALUES -(1, '2f969340-704a-4aa2-a1bd-2f832a21d257', 2, NULL, 'message 1', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), -(2, 'abe87ac1-015c-4803-be29-1e89509fe682', 2, NULL, 'message 2', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'D', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), -(3, 'a7e83a22-a6ff-4e18-82d0-19545640ccba', 2, NULL, 'message 3', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'H', 'V', 'I', '{"image/png:https://foo.bar/image1.png", "image/png:https://foo.bar/image2.png"}', NULL, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), -(4, '1cad36af-5581-4c8a-81cd-83708398f61e', 2, NULL, 'message 4', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-13 21:11:59.890662+00'), -(5, 'f557972e-2eb5-42fa-9b87-902116d18787', 3, NULL, 'message 5', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', 'I', 'H', 'V', 'I', NULL, 3, 7, 8, NULL, 1, 0, '2017-08-11 21:11:59.890662+02:00'), -(6, '579d148c-0ab1-4afb-832f-afb1fe0e19b7', 2, 2, 'message 6', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', 'I', 'H', 'V', 'I', NULL, 2, 6, 7, NULL, 1, 0, '2017-10-08 21:11:59.890662+00'), -(7, '7aeca469-2593-444e-afe4-4702317534c9', 2, NULL, 'message 7', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', 'I', 'H', 'X', 'F', NULL, 2, 6, 7, 2, 1, 0, '2018-01-02 21:11:59.890662+00'), -(9, 'e14ab466-0d3b-436d-a0f7-5851fd7d9b7d', 2, NULL, 'message 9', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'S', 'V', 'F', NULL, NULL, 6, NULL, 3, 1, 0, '2017-08-12 21:11:59.890662+00'); +(1, '2f969340-704a-4aa2-a1bd-2f832a21d257', 2, NULL, 'message 1', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'V', 'T', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), +(2, 'abe87ac1-015c-4803-be29-1e89509fe682', 2, NULL, 'message 2', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'I', 'H', 'D', 'T', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), +(3, 'a7e83a22-a6ff-4e18-82d0-19545640ccba', 2, NULL, 'message 3', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'H', 'V', 'T', '{"image/png:https://foo.bar/image1.png", "image/png:https://foo.bar/image2.png"}', NULL, 6, 7, NULL, 1, 0, '2017-08-12 21:11:59.890662+00'), +(4, '1cad36af-5581-4c8a-81cd-83708398f61e', 2, NULL, 'message 4', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', '2017-08-13 21:11:59.890662+00', 'I', 'H', 'V', 'T', NULL, 2, 6, 7, NULL, 1, 0, '2017-08-13 21:11:59.890662+00'), +(5, 'f557972e-2eb5-42fa-9b87-902116d18787', 3, NULL, 'message 5', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', '2017-08-11 21:11:59.890662+02:00', 'I', 'H', 'V', 'T', NULL, 3, 7, 8, NULL, 1, 0, '2017-08-11 21:11:59.890662+02:00'), +(6, '579d148c-0ab1-4afb-832f-afb1fe0e19b7', 2, 2, 'message 6', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', '2017-10-08 21:11:59.890662+00', 'I', 'H', 'V', 'T', NULL, 2, 6, 7, NULL, 1, 0, '2017-10-08 21:11:59.890662+00'), +(7, '7aeca469-2593-444e-afe4-4702317534c9', 2, NULL, 'message 7', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', '2018-01-02 21:11:59.890662+00', 'I', 'H', 'X', 'T', NULL, 2, 6, 7, 2, 1, 0, '2018-01-02 21:11:59.890662+00'), +(9, 'e14ab466-0d3b-436d-a0f7-5851fd7d9b7d', 2, NULL, 'message 9', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', '2017-08-12 21:11:59.890662+00', 'O', 'S', 'V', 'T', NULL, NULL, 6, NULL, 3, 1, 0, '2017-08-12 21:11:59.890662+00'); INSERT INTO msgs_label(id, uuid, name) VALUES (1, '1d9e3188-b74b-4ae0-a166-0de31aedb34a', 'Label 1'),