Updating of artifacts and artifact data (#83)

* Fixed repo interface mocks not being generated

Signed-off-by: Nick Müller <[email protected]>

* Implemented updating of artifacts
Currently only supports replacement of associated ArtifactData
Uses unreleased flyteidl update for new datacatalog endpoint

Signed-off-by: Nick Müller <[email protected]>

* Updated project to Go 1.18
Updated Dockerfile to be in line with flyteadmin
Updated GitHub automation to use Go 1.18

Signed-off-by: Nick Müller <[email protected]>

* Fixed artifact data upsert in Artifact.Update

Signed-off-by: Nick Müller <[email protected]>

* Removed golang_dockerfile from boilerplate update

Signed-off-by: Nick Müller <[email protected]>

* UpdateArtifact returns ID of artifact updated

Signed-off-by: Nick Müller <[email protected]>

* Updated to latest version of flytepropeller

Signed-off-by: Nick Müller <[email protected]>

* Updated to latest released version of flyteidl

Signed-off-by: Nick Müller <[email protected]>

* Updated to latest released flytestdlib version

Signed-off-by: Nick Müller <[email protected]>

* Missing artifact data is now removed after DB models have been updated
Prevents DB entries without underlying blob storage data in case of a partial update failure

Signed-off-by: Nick Müller <[email protected]>

Signed-off-by: Nick Müller <[email protected]>

Dockerfile

@@ -6,6 +6,16 @@
 FROM golang:1.18-alpine3.15 as builder
 RUN apk add git openssh-client make curl
 
+# Create the artifacts directory
+RUN mkdir /artifacts
+
+# Pull GRPC health probe binary for liveness and readiness checks
+RUN GRPC_HEALTH_PROBE_VERSION=v0.4.11 && \
+    wget -qO/artifacts/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /artifacts/grpc_health_probe && \
+    echo 'ded15e598d887ccc47bf2321371950bbf930f5e4856b9f75712ce4b2b5120480  /artifacts/grpc_health_probe' > .grpc_checksum && \
+    sha256sum -c .grpc_checksum
+
 # COPY only the go mod files for efficient caching
 COPY go.mod go.sum /go/src/github.com/flyteorg/datacatalog/
 WORKDIR /go/src/github.com/flyteorg/datacatalog
@@ -20,19 +30,19 @@ COPY . /go/src/github.com/flyteorg/datacatalog/
 # The main entrypoint should be compiled to /artifacts/datacatalog
 RUN make linux_compile
 
-# install grpc-health-probe 
-RUN curl --silent --fail --show-error --location --output /artifacts/grpc_health_probe "https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.5/grpc_health_probe-linux-amd64" && \
-    chmod +x /artifacts/grpc_health_probe && \
-    echo '8699c46352d752d8f533cae72728b0e65663f399fc28fb9cd854b14ad5f85f44  /artifacts/grpc_health_probe' > .grpc_checksum && \
-    sha256sum -c .grpc_checksum
-
 # update the PATH to include the /artifacts directory
 ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
-FROM alpine:3.14
+FROM alpine:3.16
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/datacatalog
+
 COPY --from=builder /artifacts /bin
 
+# Ensure the latest CA certs are present to authenticate SSL connections.
 RUN apk --update add ca-certificates
 
+RUN addgroup -S flyte && adduser -S flyte -G flyte
+USER flyte
+
 CMD ["datacatalog"]

boilerplate/update.cfg

@@ -2,6 +2,5 @@ flyte/docker_build
 flyte/golang_test_targets
 flyte/golangci_file
 flyte/golang_support_tools
-flyte/golang_dockerfile
 flyte/pull_request_template
 flyte/

go.mod

@@ -4,52 +4,54 @@ go 1.18
 
 require (
 	github.com/Selvatico/go-mocket v1.0.7
-	github.com/flyteorg/flyteidl v1.0.0
-	github.com/flyteorg/flytestdlib v1.0.5
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/flyteorg/flyteidl v1.2.3
+	github.com/flyteorg/flytestdlib v1.0.11
+	github.com/golang/glog v1.0.0
 	github.com/golang/protobuf v1.5.2
 	github.com/jackc/pgconn v1.10.1
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.1
-	google.golang.org/grpc v1.45.0
+	google.golang.org/grpc v1.46.0
 	gorm.io/driver/postgres v1.2.3
 	gorm.io/driver/sqlite v1.1.1
 	gorm.io/gorm v1.22.4
 )
 
 require (
-	cloud.google.com/go v0.100.2 // indirect
-	cloud.google.com/go/compute v1.5.0 // indirect
+	cloud.google.com/go v0.101.0 // indirect
+	cloud.google.com/go/compute v1.6.1 // indirect
 	cloud.google.com/go/iam v0.3.0 // indirect
-	cloud.google.com/go/storage v1.14.0 // indirect
-	github.com/Azure/azure-sdk-for-go v62.3.0+incompatible // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect
+	cloud.google.com/go/storage v1.22.0 // indirect
+	github.com/Azure/azure-sdk-for-go v63.4.0+incompatible // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.25 // indirect
+	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/aws/aws-sdk-go v1.43.37 // indirect
+	github.com/aws/aws-sdk-go v1.44.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/coocood/freecache v1.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/flyteorg/stow v0.3.4 // indirect
+	github.com/flyteorg/stow v0.3.6 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
+	github.com/googleapis/go-type-adapters v1.0.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -84,16 +86,16 @@ require (
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
+	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
-	google.golang.org/api v0.74.0 // indirect
+	google.golang.org/api v0.76.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
+	google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect