diff --git a/go.mod b/go.mod index 3e6a2949..a08d1bca 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,21 @@ go 1.19 require ( github.com/Selvatico/go-mocket v1.0.7 github.com/flyteorg/flyteidl v1.3.6 - github.com/flyteorg/flytestdlib v1.0.15 + github.com/flyteorg/flytestdlib v1.0.17-0.20230320195919-90331d171e2a + github.com/go-gormigrate/gormigrate/v2 v2.0.2 + github.com/go-sql-driver/mysql v1.7.0 github.com/gofrs/uuid v4.2.0+incompatible github.com/golang/glog v1.0.0 github.com/golang/protobuf v1.5.2 - github.com/jackc/pgconn v1.10.1 + github.com/jackc/pgconn v1.13.0 github.com/mitchellh/mapstructure v1.4.3 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.0 google.golang.org/grpc v1.46.0 - gorm.io/driver/postgres v1.2.3 - gorm.io/driver/sqlite v1.1.1 - gorm.io/gorm v1.22.4 + gorm.io/driver/postgres v1.4.5 + gorm.io/driver/sqlite v1.4.4 + gorm.io/gorm v1.25.0 ) require ( @@ -56,17 +58,17 @@ require ( github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.1 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.9.0 // indirect - github.com/jackc/pgx/v4 v4.14.0 // indirect + github.com/jackc/pgtype v1.12.0 // indirect + github.com/jackc/pgx/v4 v4.17.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jinzhu/now v1.1.4 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/mattn/go-sqlite3 v1.14.0 // indirect + github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/ncw/swift v1.0.53 // indirect github.com/pelletier/go-toml v1.9.4 // indirect @@ -82,10 +84,10 @@ require ( github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.11.0 // indirect - github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect @@ -98,7 +100,8 @@ require ( google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/mysql v1.5.0 // indirect k8s.io/apimachinery v0.20.2 // indirect k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect k8s.io/klog/v2 v2.5.0 // indirect diff --git a/go.sum b/go.sum index cde10c1b..c8b6183c 100644 --- a/go.sum +++ b/go.sum @@ -96,7 +96,6 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Selvatico/go-mocket v1.0.7 h1:sXuFMnMfVL9b/Os8rGXPgbOFbr4HJm8aHsulD/uMTUk= @@ -106,7 +105,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.2 h1:5VBk5r06bgxgRKVaUtm1/4NT/rtrnH2E4cnAYv5zgQc= @@ -146,6 +144,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.12.0 h1:VtrkII767ttSPNRfFekePK3sctr+joXgO58stqQbtUA= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -167,8 +166,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/flyteorg/flyteidl v1.3.6 h1:PI846AdnrQZ84pxRVAzA3WGihv+xXmjQHO91nj/kV9g= github.com/flyteorg/flyteidl v1.3.6/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM= -github.com/flyteorg/flytestdlib v1.0.15 h1:kv9jDQmytbE84caY+pkZN8trJU2ouSAmESzpTEhfTt0= -github.com/flyteorg/flytestdlib v1.0.15/go.mod h1:ghw/cjY0sEWIIbyCtcJnL/Gt7ZS7gf9SUi0CCPhbz3s= +github.com/flyteorg/flytestdlib v1.0.17-0.20230320195919-90331d171e2a h1:WQTudPI+ysEkMp648YDm8lbrVFh0sE5K5zuwjFNB37Y= +github.com/flyteorg/flytestdlib v1.0.17-0.20230320195919-90331d171e2a/go.mod h1:TcKdywJD/EokpPaTeRyn0rY0ErXKPzSsg0JKd4U0xDg= github.com/flyteorg/stow v0.3.6 h1:jt50ciM14qhKBaIrB+ppXXY+SXB59FNREFgTJqCyqIk= github.com/flyteorg/stow v0.3.6/go.mod h1:5dfBitPM004dwaZdoVylVjxFT4GWAgI0ghAndhNUzCo= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -182,6 +181,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gormigrate/gormigrate/v2 v2.0.2 h1:YV4Lc5yMQX8ahVW0ENPq6sPhrhdkGukc6fPRYmZ1R6Y= +github.com/go-gormigrate/gormigrate/v2 v2.0.2/go.mod h1:vld36QpBTfTzLealsHsmQQJK5lSwJt6wiORv+oFX8/I= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -199,6 +200,8 @@ github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL9 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= @@ -211,6 +214,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188 h1:+eHOFJl1BaXrQxKX+T06f78590z4qA2ZzBTqahsKSE4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -326,8 +331,8 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8= -github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= +github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -343,37 +348,36 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= -github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= +github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.9.0 h1:/SH1RxEtltvJgsDqp3TbiTFApD3mey3iygpuEGeuBXk= -github.com/jackc/pgtype v1.9.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= +github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.14.0 h1:TgdrmgnM7VY72EuSQzBbBd4JA1RLqJolrw9nQVZABVc= -github.com/jackc/pgx/v4 v4.14.0/go.mod h1:jT3ibf/A0ZVCp89rtCIN0zCJxcE74ypROmHEZYsG/j8= +github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= +github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -419,8 +423,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -514,16 +518,17 @@ github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUs github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= -github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -569,8 +574,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -606,7 +611,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1044,16 +1048,20 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.2.3 h1:f4t0TmNMy9gh3TU2PX+EppoA6YsgFnyq8Ojtddb42To= -gorm.io/driver/postgres v1.2.3/go.mod h1:pJV6RgYQPG47aM1f0QeOzFH9HxQc8JcmAgjRCgS0wjs= -gorm.io/driver/sqlite v1.1.1 h1:qtWqNAEUyi7gYSUAJXeiAMz0lUOdakZF5ia9Fqnp5G4= -gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI= -gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.4 h1:8aPcyEJhY0MAt8aY6Dc524Pn+pO29K+ydu+e/cXSpQM= -gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.5.0 h1:6hSAT5QcyIaty0jfnff0z0CLDjyRgZ8mlMHLqSt7uXM= +gorm.io/driver/mysql v1.5.0/go.mod h1:FFla/fJuCvyTi7rJQd27qlNX2v3L6deTR1GgTjSOLPo= +gorm.io/driver/postgres v1.4.5 h1:mTeXTTtHAgnS9PgmhN2YeUbazYpLhUI1doLnw42XUZc= +gorm.io/driver/postgres v1.4.5/go.mod h1:GKNQYSJ14qvWkvPwXljMGehpKrhlDNsqYRr5HnYGncg= +gorm.io/driver/sqlite v1.4.4 h1:gIufGoR0dQzjkyqDyYSCvsYR6fba1Gw5YKDqKeChxFc= +gorm.io/driver/sqlite v1.4.4/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= +gorm.io/driver/sqlserver v1.3.2 h1:yYt8f/xdAKLY7lCCyXxIUEgZ/WsURos3dHrx8MKFGAk= +gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.1-0.20221019064659-5dd2bb482755/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU= +gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/repositories/config/migrations.go b/pkg/repositories/config/migrations.go new file mode 100644 index 00000000..4bf102a1 --- /dev/null +++ b/pkg/repositories/config/migrations.go @@ -0,0 +1,19 @@ +package config + +import ( + fixupmigrations "github.com/flyteorg/datacatalog/pkg/repositories/config/migrations/fixup" + noopmigrations "github.com/flyteorg/datacatalog/pkg/repositories/config/migrations/noop" + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + + +func ListMigrations(db *gorm.DB) []*gormigrate.Migration { + var Migrations []*gormigrate.Migration + if db.Dialector.Name() == "postgres" { + Migrations = append(Migrations, noopmigrations.Migrations...) + } + Migrations = append(Migrations, fixupmigrations.Migrations...) + return Migrations +} + diff --git a/pkg/repositories/config/migrations/fixup/migrations.go b/pkg/repositories/config/migrations/fixup/migrations.go new file mode 100644 index 00000000..b0206c30 --- /dev/null +++ b/pkg/repositories/config/migrations/fixup/migrations.go @@ -0,0 +1,194 @@ +package fixupmigrations + +import ( + "time" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" + "gorm.io/gorm/schema" +) + +type UUIDString string + +func (uuidString UUIDString) GormDBDataType(db *gorm.DB, field *schema.Field) string { + // use field.Tag, field.TagSettings gets field's tags + // checkout https://github.com/go-gorm/gorm/blob/master/schema/field.go for all options + if db.Dialector.Name() == "mysql" { + return "varchar(36)" + } + return "uuid" +} + +type BaseModel struct { + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time `sql:"index"` +} + +type ArtifactKey struct { + DatasetProject string `gorm:"size:64;primary_key"` + // Can we use a smaller size fo dataset name? This is `flyte-task-` + DatasetName string `gorm:"size:100;primary_key"` + DatasetDomain string `gorm:"size:64;primary_key"` + DatasetVersion string `gorm:"size:128;primary_key"` + // This is a UUID + ArtifactID string `gorm:"size:36;primary_key"` +} + +type Artifact struct { + BaseModel + ArtifactKey + DatasetUUID UUIDString `gorm:"type:uuid;index:artifacts_dataset_uuid_idx"` + Dataset Dataset `gorm:"association_autocreate:false"` + ArtifactData []ArtifactData `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` + Partitions []Partition `gorm:"references:ArtifactID;foreignkey:ArtifactID"` + Tags []Tag `gorm:"references:ArtifactID,DatasetUUID;foreignkey:ArtifactID,DatasetUUID"` + SerializedMetadata []byte +} + +type ArtifactData struct { + BaseModel + ArtifactKey + Name string `gorm:"size:32;primary_key"` + Location string `gorm:"size:2048"` +} + + +type DatasetKey struct { + Project string `gorm:"size:64;primary_key"` // part of pkey, no index needed as it is first column in the pkey + // TODO: figure out what size this should be + Name string `gorm:"size:100;primary_key;index:dataset_name_idx"` // part of pkey and has separate index for filtering + Domain string `gorm:"size:64;primary_key;index:dataset_domain_idx"` // part of pkey and has separate index for filtering + Version string `gorm:"size:128;primary_key;index:dataset_version_idx"` // part of pkey and has separate index for filtering + UUID UUIDString `gorm:"unique;type:uuid"` +} + +type Dataset struct { + BaseModel + DatasetKey + SerializedMetadata []byte + PartitionKeys []PartitionKey `gorm:"references:UUID;foreignkey:DatasetUUID"` +} + +type PartitionKey struct { + BaseModel + DatasetUUID UUIDString `gorm:"type:uuid;primary_key"` + // TODO: figure out if this is used. + Name string `gorm:"size:100;primary_key"` +} + +type TagKey struct { + DatasetProject string `gorm:"size:64;primary_key"` + // TODO: figure out what size this should be + DatasetName string `gorm:"size:100;primary_key"` + DatasetDomain string `gorm:"size:64;primary_key"` + DatasetVersion string `gorm:"size:128;primary_key"` + TagName string `gorm:"size:56;primary_key"` +} + +type Tag struct { + BaseModel + TagKey + ArtifactID string `gorm:"size:36"` + DatasetUUID UUIDString `gorm:"index:tags_dataset_uuid_idx"` + Artifact Artifact `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` +} + +// TODO: figure out if this is used. +type Partition struct { + BaseModel + DatasetUUID UUIDString `gorm:"primary_key;type:uuid"` + Key string `gorm:"primary_key"` + Value string `gorm:"primary_key"` + ArtifactID string `gorm:"primary_key;index"` // index for JOINs with the Tag/Labels table when querying artifacts +} + +type ReservationKey struct { + DatasetProject string `gorm:"size:64;primary_key"` + // TODO: figure out what size this should be + DatasetName string `gorm:"size:100;primary_key"` + DatasetDomain string `gorm:"size:64;primary_key"` + DatasetVersion string `gorm:"size:128;primary_key"` + // TODO: figure out what size this should be + TagName string `gorm:"56;primary_key"` +} + +// Reservation tracks the metadata needed to allow +// task cache serialization +type Reservation struct { + BaseModel + ReservationKey + + // Identifies who owns the reservation + OwnerID string + + // When the reservation will expire + ExpiresAt time.Time + SerializedMetadata []byte +} + +var Migrations = []*gormigrate.Migration{ + { + ID: "2023-04-18-fixup-dataset", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Dataset{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-fixup-artifact", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Artifact{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-fixup-artifact-data", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&ArtifactData{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-fixup-tag", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Tag{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-fixup-partition-key", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&PartitionKey{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-fixup-partition", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Partition{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-fixup-reservation", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Reservation{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, +} diff --git a/pkg/repositories/config/migrations/noop/migrations.go b/pkg/repositories/config/migrations/noop/migrations.go new file mode 100644 index 00000000..904567f4 --- /dev/null +++ b/pkg/repositories/config/migrations/noop/migrations.go @@ -0,0 +1,186 @@ +package noopmigrations + +import ( + "time" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" + "gorm.io/gorm/schema" +) + +type UUIDString string + +func (uuidString UUIDString) GormDBDataType(db *gorm.DB, field *schema.Field) string { + // use field.Tag, field.TagSettings gets field's tags + // checkout https://github.com/go-gorm/gorm/blob/master/schema/field.go for all options + if db.Dialector.Name() == "mysql" { + return "varchar(36)" + } + return "uuid" +} + +type BaseModel struct { + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time `sql:"index"` +} + +type ArtifactKey struct { + DatasetProject string `gorm:"primary_key"` + DatasetName string `gorm:"primary_key"` + DatasetDomain string `gorm:"primary_key"` + DatasetVersion string `gorm:"primary_key"` + ArtifactID string `gorm:"primary_key"` +} + +type Artifact struct { + BaseModel + ArtifactKey + DatasetUUID UUIDString `gorm:"type:uuid;index:artifacts_dataset_uuid_idx"` + Dataset Dataset `gorm:"association_autocreate:false"` + ArtifactData []ArtifactData `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` + Partitions []Partition `gorm:"references:ArtifactID;foreignkey:ArtifactID"` + Tags []Tag `gorm:"references:ArtifactID,DatasetUUID;foreignkey:ArtifactID,DatasetUUID"` + SerializedMetadata []byte +} + +type ArtifactData struct { + BaseModel + ArtifactKey + Name string `gorm:"primary_key"` + Location string +} + + +type DatasetKey struct { + Project string `gorm:"primary_key;"` // part of pkey, no index needed as it is first column in the pkey + Name string `gorm:"primary_key;index:dataset_name_idx"` // part of pkey and has separate index for filtering + Domain string `gorm:"primary_key;index:dataset_domain_idx"` // part of pkey and has separate index for filtering + Version string `gorm:"primary_key;index:dataset_version_idx"` // part of pkey and has separate index for filtering + UUID UUIDString `gorm:"unique;type:uuid"` +} + +type Dataset struct { + BaseModel + DatasetKey + SerializedMetadata []byte + PartitionKeys []PartitionKey `gorm:"references:UUID;foreignkey:DatasetUUID"` +} + +type PartitionKey struct { + BaseModel + DatasetUUID UUIDString `gorm:"type:uuid;primary_key"` + Name string `gorm:"primary_key"` +} + +type TagKey struct { + DatasetProject string `gorm:"primary_key"` + DatasetName string `gorm:"primary_key"` + DatasetDomain string `gorm:"primary_key"` + DatasetVersion string `gorm:"primary_key"` + TagName string `gorm:"primary_key"` +} + +type Tag struct { + BaseModel + TagKey + ArtifactID string + DatasetUUID UUIDString `gorm:"index:tags_dataset_uuid_idx"` + Artifact Artifact `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` +} + +type Partition struct { + BaseModel + DatasetUUID UUIDString `gorm:"primary_key;type:uuid"` + Key string `gorm:"primary_key"` + Value string `gorm:"primary_key"` + ArtifactID string `gorm:"primary_key;index"` // index for JOINs with the Tag/Labels table when querying artifacts +} + +type ReservationKey struct { + DatasetProject string `gorm:"primary_key"` + DatasetName string `gorm:"primary_key"` + DatasetDomain string `gorm:"primary_key"` + DatasetVersion string `gorm:"primary_key"` + TagName string `gorm:"primary_key"` +} + +// Reservation tracks the metadata needed to allow +// task cache serialization +type Reservation struct { + BaseModel + ReservationKey + + // Identifies who owns the reservation + OwnerID string + + // When the reservation will expire + ExpiresAt time.Time + SerializedMetadata []byte +} + +var Migrations = []*gormigrate.Migration{ + { + ID: "2023-04-18-noop-dataset", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Dataset{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-noop-artifact", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Artifact{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-noop-artifact-data", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&ArtifactData{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-noop-tag", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Tag{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-noop-partition-key", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&PartitionKey{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-noop-partition", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Partition{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, + { + ID: "2023-04-18-noop-reservation", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Reservation{}) + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, + }, +} diff --git a/pkg/repositories/config/postgres.go b/pkg/repositories/config/postgres.go deleted file mode 100644 index bcfb428a..00000000 --- a/pkg/repositories/config/postgres.go +++ /dev/null @@ -1,93 +0,0 @@ -package config - -import ( - "context" - "fmt" - - "github.com/flyteorg/flytestdlib/database" - stdlibLogger "github.com/flyteorg/flytestdlib/logger" - "github.com/flyteorg/flytestdlib/promutils" - - "gorm.io/driver/postgres" - "gorm.io/gorm" - "gorm.io/gorm/logger" -) - -const ( - Postgres = "postgres" - Sqlite = "sqlite" -) - -// Generic interface for providing a config necessary to open a database connection. -type DbConnectionConfigProvider interface { - // Returns database dialector - GetDialector() gorm.Dialector - - GetDBConfig() database.DbConfig - - GetDSN() string -} - -type BaseConfig struct { - LogLevel logger.LogLevel `json:"log_level"` - DisableForeignKeyConstraintWhenMigrating bool -} - -// PostgreSQL implementation for DbConnectionConfigProvider. -type PostgresConfigProvider struct { - config database.DbConfig - scope promutils.Scope -} - -// TODO : Make the Config provider itself env based -func NewPostgresConfigProvider(config database.DbConfig, scope promutils.Scope) DbConnectionConfigProvider { - return &PostgresConfigProvider{ - config: config, - scope: scope, - } -} - -func (p *PostgresConfigProvider) GetDSN() string { - if p.config.Postgres.Password == "" { - // Switch for development - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s sslmode=disable", - p.config.Postgres.Host, p.config.Postgres.Port, p.config.Postgres.DbName, p.config.Postgres.User) - } - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s %s", - p.config.Postgres.Host, p.config.Postgres.Port, p.config.Postgres.DbName, p.config.Postgres.User, p.config.Postgres.Password, p.config.Postgres.ExtraOptions) -} - -func (p *PostgresConfigProvider) GetDialector() gorm.Dialector { - return postgres.Open(p.GetDSN()) -} - -func (p *PostgresConfigProvider) GetDBConfig() database.DbConfig { - return p.config -} - -// Opens a connection to the database specified in the config. -// You must call CloseDbConnection at the end of your session! -func OpenDbConnection(ctx context.Context, config DbConnectionConfigProvider) (*gorm.DB, error) { - dbConfig := config.GetDBConfig() - - db, err := gorm.Open(config.GetDialector(), &gorm.Config{ - Logger: database.GetGormLogger(ctx, stdlibLogger.GetConfig()), - DisableForeignKeyConstraintWhenMigrating: !dbConfig.EnableForeignKeyConstraintWhenMigrating, - }) - if err != nil { - return nil, err - } - - return db, setupDbConnectionPool(db, &dbConfig) -} - -func setupDbConnectionPool(gormDb *gorm.DB, dbConfig *database.DbConfig) error { - genericDb, err := gormDb.DB() - if err != nil { - return err - } - genericDb.SetConnMaxLifetime(dbConfig.ConnMaxLifeTime.Duration) - genericDb.SetMaxIdleConns(dbConfig.MaxIdleConnections) - genericDb.SetMaxOpenConns(dbConfig.MaxOpenConnections) - return nil -} diff --git a/pkg/repositories/config/postgres_test.go b/pkg/repositories/config/postgres_test.go deleted file mode 100644 index 659b7d72..00000000 --- a/pkg/repositories/config/postgres_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/flyteorg/flytestdlib/config" - "github.com/flyteorg/flytestdlib/database" - mockScope "github.com/flyteorg/flytestdlib/promutils" - - "github.com/stretchr/testify/assert" - - "gorm.io/driver/sqlite" - "gorm.io/gorm" -) - -func TestConstructGormArgs(t *testing.T) { - postgresConfigProvider := NewPostgresConfigProvider(database.DbConfig{Postgres: database.PostgresConfig{ - Host: "localhost", - Port: 5432, - DbName: "postgres", - User: "postgres", - ExtraOptions: "sslmode=disable", - }, - EnableForeignKeyConstraintWhenMigrating: false, - }, mockScope.NewTestScope()) - - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres sslmode=disable", postgresConfigProvider.GetDSN()) - assert.Equal(t, false, postgresConfigProvider.GetDBConfig().EnableForeignKeyConstraintWhenMigrating) -} - -func TestConstructGormArgsWithPassword(t *testing.T) { - postgresConfigProvider := NewPostgresConfigProvider(database.DbConfig{Postgres: database.PostgresConfig{ - Host: "localhost", - Port: 5432, - DbName: "postgres", - User: "postgres", - Password: "pass", - ExtraOptions: "sslmode=enable", - }, - }, mockScope.NewTestScope()) - - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass sslmode=enable", postgresConfigProvider.GetDSN()) -} - -func TestConstructGormArgsWithPasswordNoExtra(t *testing.T) { - postgresConfigProvider := NewPostgresConfigProvider(database.DbConfig{Postgres: database.PostgresConfig{ - Host: "localhost", - Port: 5432, - DbName: "postgres", - User: "postgres", - Password: "pass", - }, - }, mockScope.NewTestScope()) - - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass ", postgresConfigProvider.GetDSN()) -} - -func TestSetupDbConnectionPool(t *testing.T) { - t.Run("successful", func(t *testing.T) { - gormDb, err := gorm.Open(sqlite.Open(filepath.Join(os.TempDir(), "gorm.db")), &gorm.Config{}) - assert.Nil(t, err) - dbConfig := &database.DbConfig{ - DeprecatedPort: 5432, - MaxIdleConnections: 10, - MaxOpenConnections: 1000, - ConnMaxLifeTime: config.Duration{Duration: time.Hour}, - } - err = setupDbConnectionPool(gormDb, dbConfig) - assert.Nil(t, err) - genericDb, err := gormDb.DB() - assert.Nil(t, err) - assert.Equal(t, genericDb.Stats().MaxOpenConnections, 1000) - }) - t.Run("unset duration", func(t *testing.T) { - gormDb, err := gorm.Open(sqlite.Open(filepath.Join(os.TempDir(), "gorm.db")), &gorm.Config{}) - assert.Nil(t, err) - dbConfig := &database.DbConfig{ - DeprecatedPort: 5432, - MaxIdleConnections: 10, - MaxOpenConnections: 1000, - } - err = setupDbConnectionPool(gormDb, dbConfig) - assert.Nil(t, err) - genericDb, err := gormDb.DB() - assert.Nil(t, err) - assert.Equal(t, genericDb.Stats().MaxOpenConnections, 1000) - }) - t.Run("failed to get DB", func(t *testing.T) { - gormDb := &gorm.DB{ - Config: &gorm.Config{ - ConnPool: &gorm.PreparedStmtDB{}, - }, - } - dbConfig := &database.DbConfig{ - DeprecatedPort: 5432, - MaxIdleConnections: 10, - MaxOpenConnections: 1000, - ConnMaxLifeTime: config.Duration{Duration: time.Hour}, - } - err := setupDbConnectionPool(gormDb, dbConfig) - assert.NotNil(t, err) - }) -} diff --git a/pkg/repositories/database.go b/pkg/repositories/database.go new file mode 100644 index 00000000..d22f794d --- /dev/null +++ b/pkg/repositories/database.go @@ -0,0 +1,28 @@ +package repositories + +import ( + "github.com/flyteorg/flytestdlib/database" + stdlibLogger "github.com/flyteorg/flytestdlib/logger" + "gorm.io/gorm" + + "context" +) + +const ( + Postgres = "postgres" + Sqlite = "sqlite" +) + +// OpenDbConnection opens a connection to the database specified in the config. +// You must call CloseDbConnection at the end of your session! +func OpenDbConnection(ctx context.Context, dbConfig database.DbConfig) (*gorm.DB, error) { + gormConfig := &gorm.Config{ + Logger: database.GetGormLogger(ctx, stdlibLogger.GetConfig()), + DisableForeignKeyConstraintWhenMigrating: !dbConfig.EnableForeignKeyConstraintWhenMigrating, + } + db, err := NewDBHandle(ctx, dbConfig, gormConfig) + if err != nil { + return nil, err + } + return db.db, nil +} diff --git a/pkg/repositories/errors/generic.go b/pkg/repositories/errors/generic.go new file mode 100644 index 00000000..2532db02 --- /dev/null +++ b/pkg/repositories/errors/generic.go @@ -0,0 +1,37 @@ +package errors + +import ( + "errors" + "reflect" + + "github.com/flyteorg/flytestdlib/logger" + + catalogErrors "github.com/flyteorg/datacatalog/pkg/errors" + "google.golang.org/grpc/codes" + "gorm.io/gorm" +) + +type genericErrorTransformer struct { +} + +func (p *genericErrorTransformer) fromGormError(err error) error { + switch err.Error() { + case gorm.ErrRecordNotFound.Error(): + return catalogErrors.NewDataCatalogErrorf(codes.NotFound, "entry not found") + default: + logger.InfofNoCtx("Generic error detected. Error type: [%v]", reflect.TypeOf(err)) + return catalogErrors.NewDataCatalogErrorf(codes.Internal, unexpectedType, err) + } +} + +func (p *genericErrorTransformer) ToDataCatalogError(err error) error { + if unwrappedErr := errors.Unwrap(err); unwrappedErr != nil { + err = unwrappedErr + } + + return p.fromGormError(err) +} + +func NewGenericErrorTransformer() ErrorTransformer { + return &genericErrorTransformer{} +} diff --git a/pkg/repositories/errors/mysql.go b/pkg/repositories/errors/mysql.go new file mode 100644 index 00000000..ee07b850 --- /dev/null +++ b/pkg/repositories/errors/mysql.go @@ -0,0 +1,70 @@ +package errors + +import ( + "errors" + "fmt" + "reflect" + + "github.com/flyteorg/flytestdlib/logger" + "github.com/go-sql-driver/mysql" + + catalogErrors "github.com/flyteorg/datacatalog/pkg/errors" + "google.golang.org/grpc/codes" + "gorm.io/gorm" +) + +// MySql error codes +const ( + duplicateKeyError = 1062 + // uniqueConstraintViolationCode = "23505" + // undefinedTable = "42P01" +) + +type mysqlErrorTransformer struct { +} + +const ( + duplicateKeyErrorFormat = "duplicate key value violates unique constraint %s" + defaultMysqlError = "failed database operation with code [%s] and msg [%s]" + // unexpectedType = "unexpected error type for: %v" + // uniqueConstraintViolation = "value with matching already exists (%s)" + // defaultPgError = "failed database operation with code [%s] and msg [%s]" + // unsupportedTableOperation = "cannot query with specified table attributes: %s" +) + +func (p *mysqlErrorTransformer) fromGormError(err error) error { + switch err.Error() { + case gorm.ErrRecordNotFound.Error(): + return catalogErrors.NewDataCatalogErrorf(codes.NotFound, "entry not found") + default: + return catalogErrors.NewDataCatalogErrorf(codes.Internal, unexpectedType, err) + } +} + +func (p *mysqlErrorTransformer) ToDataCatalogError(err error) error { + if unwrappedErr := errors.Unwrap(err); unwrappedErr != nil { + err = unwrappedErr + } + + mysqlError, ok := err.(*mysql.MySQLError) + if !ok { + logger.InfofNoCtx("Unable to cast to mysql.MySQLError. Error type: [%v]", + reflect.TypeOf(err)) + return p.fromGormError(err) + } + + switch mysqlError.Number { + case 1062: + return catalogErrors.NewDataCatalogErrorf(codes.AlreadyExists, duplicateKeyErrorFormat, mysqlError.Message) + // case uniqueConstraintViolationCode: + // return catalogErrors.NewDataCatalogErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Message) + // case undefinedTable: + // return catalogErrors.NewDataCatalogErrorf(codes.InvalidArgument, unsupportedTableOperation, pqError.Message) + default: + return catalogErrors.NewDataCatalogErrorf(codes.Unknown, fmt.Sprintf(defaultMysqlError, mysqlError.Number, mysqlError.Message)) + } +} + +func NewMySqlErrorTransformer() ErrorTransformer { + return &mysqlErrorTransformer{} +} diff --git a/pkg/repositories/errors/postgres.go b/pkg/repositories/errors/postgres.go index 39cdddcc..ee689db2 100644 --- a/pkg/repositories/errors/postgres.go +++ b/pkg/repositories/errors/postgres.go @@ -64,8 +64,3 @@ func (p *postgresErrorTransformer) ToDataCatalogError(err error) error { func NewPostgresErrorTransformer() ErrorTransformer { return &postgresErrorTransformer{} } - -type ConnectError interface { - Unwrap() error - Error() string -} diff --git a/pkg/repositories/factory.go b/pkg/repositories/factory.go index 92b18ac3..ab117fad 100644 --- a/pkg/repositories/factory.go +++ b/pkg/repositories/factory.go @@ -2,26 +2,13 @@ package repositories import ( "context" - "fmt" - "github.com/flyteorg/flytestdlib/database" - "github.com/flyteorg/datacatalog/pkg/repositories/config" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/flytestdlib/promutils" ) -type RepoConfig int32 - -const ( - POSTGRES RepoConfig = 0 -) - -var RepositoryConfigurationName = map[RepoConfig]string{ - POSTGRES: "POSTGRES", -} - // The RepositoryInterface indicates the methods that each Repository must support. // A Repository indicates a Database which is collection of Tables/models. // The goal is allow databases to be Plugged in easily. @@ -32,18 +19,21 @@ type RepositoryInterface interface { ReservationRepo() interfaces.ReservationRepo } -func GetRepository(ctx context.Context, repoType RepoConfig, dbConfig database.DbConfig, scope promutils.Scope) RepositoryInterface { - switch repoType { - case POSTGRES: - db, err := config.OpenDbConnection(ctx, config.NewPostgresConfigProvider(dbConfig, scope.NewSubScope("postgres"))) - if err != nil { - panic(err) - } - return NewPostgresRepo( - db, - errors.NewPostgresErrorTransformer(), - scope.NewSubScope("repositories")) - default: - panic(fmt.Sprintf("Invalid repoType %v", repoType)) +func GetRepository(ctx context.Context, dbConfig database.DbConfig, scope promutils.Scope) RepositoryInterface { + db, err := OpenDbConnection(ctx, dbConfig) + if err != nil { + panic(err) + } + + var errTransformer errors.ErrorTransformer + if !dbConfig.Mysql.IsEmpty() { + errTransformer = errors.NewMySqlErrorTransformer() + return NewMySqlRepo(db, errTransformer, scope.NewSubScope("repositories")) + } else if !dbConfig.Postgres.IsEmpty() { + errTransformer = errors.NewPostgresErrorTransformer() + return NewPostgresRepo(db, errTransformer, scope.NewSubScope("repositories")) + } else { + errTransformer = errors.NewGenericErrorTransformer() } + panic("Unsupported database type") } diff --git a/pkg/repositories/handle.go b/pkg/repositories/handle.go index ff404ba3..980ecc53 100644 --- a/pkg/repositories/handle.go +++ b/pkg/repositories/handle.go @@ -9,10 +9,8 @@ import ( "github.com/flyteorg/flytestdlib/database" - "github.com/flyteorg/datacatalog/pkg/repositories/config" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/logger" - "github.com/flyteorg/flytestdlib/promutils" "gorm.io/gorm" ) @@ -20,15 +18,24 @@ type DBHandle struct { db *gorm.DB } -func NewDBHandle(ctx context.Context, dbConfigValues database.DbConfig, catalogScope promutils.Scope) (*DBHandle, error) { +func NewDBHandle(ctx context.Context, dbConfigValues database.DbConfig, gormConfig *gorm.Config) (*DBHandle, error) { var gormDb *gorm.DB var err error + logConfig := logger.GetConfig() + if gormConfig == nil { + gormConfig = &gorm.Config{ + Logger: database.GetGormLogger(ctx, logConfig), + DisableForeignKeyConstraintWhenMigrating: true, + } + } switch { case !dbConfigValues.SQLite.IsEmpty(): gormDb, err = gorm.Open(sqlite.Open(dbConfigValues.SQLite.File)) + case !dbConfigValues.Mysql.IsEmpty(): + gormDb, err = database.CreateMysqlDbIfNotExists(ctx, gormConfig, dbConfigValues.Mysql) case !dbConfigValues.Postgres.IsEmpty(): - gormDb, err = config.OpenDbConnection(ctx, config.NewPostgresConfigProvider(dbConfigValues, catalogScope.NewSubScope(config.Postgres))) + gormDb, err = database.CreatePostgresDbIfNotExists(ctx, gormConfig, dbConfigValues.Postgres) default: return nil, fmt.Errorf("unrecognized database config, %v. Supported only postgres and sqlite", dbConfigValues) } @@ -44,41 +51,12 @@ func NewDBHandle(ctx context.Context, dbConfigValues database.DbConfig, catalogS return out, nil } -func (h *DBHandle) CreateDB(dbName string) error { - type DatabaseResult struct { - Exists bool - } - var checkExists DatabaseResult - result := h.db.Raw("SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = ?)", dbName).Scan(&checkExists) - if result.Error != nil { - return result.Error - } - - // create db if it does not exist - if !checkExists.Exists { - logger.Infof(context.TODO(), "Creating Database %v since it does not exist", dbName) - - // NOTE: golang sql drivers do not support parameter injection for CREATE calls - createDBStatement := fmt.Sprintf("CREATE DATABASE %s", dbName) - result = h.db.Exec(createDBStatement) - - if result.Error != nil { - if !isPgErrorWithCode(result.Error, pqDbAlreadyExistsCode) { - return result.Error - } - logger.Infof(context.TODO(), "Not creating database %s, already exists", dbName) - } - } - - return nil -} - func (h *DBHandle) Migrate(ctx context.Context) error { if err := h.db.AutoMigrate(&models.Dataset{}); err != nil { return err } - if err := h.db.Debug().AutoMigrate(&models.Artifact{}); err != nil { + if err := h.db.AutoMigrate(&models.Artifact{}); err != nil { return err } diff --git a/pkg/repositories/handle_test.go b/pkg/repositories/handle_test.go index a412e678..725d69e7 100644 --- a/pkg/repositories/handle_test.go +++ b/pkg/repositories/handle_test.go @@ -5,93 +5,20 @@ import ( "path" "testing" - mocket "github.com/Selvatico/go-mocket" - "github.com/flyteorg/datacatalog/pkg/repositories/config" "github.com/flyteorg/flytestdlib/database" "github.com/stretchr/testify/assert" - - "database/sql/driver" - - "github.com/flyteorg/datacatalog/pkg/repositories/utils" ) -func TestCreateDB(t *testing.T) { - GlobalMock := mocket.Catcher.Reset() - GlobalMock.Logging = true - - checkExists := false - GlobalMock.NewMock().WithQuery( - `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = $1)%!(EXTRA string=testDB)`).WithCallback( - func(s string, values []driver.NamedValue) { - checkExists = true - }, - ).WithReply([]map[string]interface{}{ - {"exists": false}, - }) - - createdDB := false - - // NOTE: unfortunately mocket does not support checking CREATE statements, but let's match the suffix - GlobalMock.NewMock().WithQuery( - `DATABASE testDB`).WithCallback( - func(s string, values []driver.NamedValue) { - assert.Equal(t, "CREATE DATABASE testDB", s) - createdDB = true - }, - ) - - db := utils.GetDbForTest(t) - dbHandle := &DBHandle{ - db: db, - } - _ = dbHandle.CreateDB("testDB") - assert.True(t, checkExists) - assert.True(t, createdDB) -} - -func TestDBAlreadyExists(t *testing.T) { - GlobalMock := mocket.Catcher.Reset() - GlobalMock.Logging = true - - checkExists := false - GlobalMock.NewMock().WithQuery( - `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = $1)%!(EXTRA string=testDB)`).WithCallback( - func(s string, values []driver.NamedValue) { - checkExists = true - }, - ).WithReply([]map[string]interface{}{ - {"exists": true}, - }) - - createdDB := false - GlobalMock.NewMock().WithQuery( - `DATABASE testDB`).WithCallback( - func(s string, values []driver.NamedValue) { - createdDB = false - }, - ) - - db := utils.GetDbForTest(t) - dbHandle := &DBHandle{ - db: db, - } - err := dbHandle.CreateDB("testDB") - assert.NoError(t, err) - assert.True(t, checkExists) - assert.False(t, createdDB) -} - func TestNewDBHandle(t *testing.T) { t.Run("missing DB Config", func(t *testing.T) { - _, err := NewDBHandle(context.TODO(), database.DbConfig{}, migrateScope) + _, err := NewDBHandle(context.TODO(), database.DbConfig{}, nil) assert.Error(t, err) }) t.Run("sqlite config", func(t *testing.T) { dbFile := path.Join(t.TempDir(), "admin.db") - dbHandle, err := NewDBHandle(context.TODO(), database.DbConfig{SQLite: database.SQLiteConfig{File: dbFile}}, migrateScope) + dbHandle, err := NewDBHandle(context.TODO(), database.DbConfig{SQLite: database.SQLiteConfig{File: dbFile}}, nil) assert.Nil(t, err) assert.NotNil(t, dbHandle) - assert.Equal(t, config.Sqlite, dbHandle.db.Name()) }) } diff --git a/pkg/repositories/initialize.go b/pkg/repositories/initialize.go index ff6b5a79..99c8a9ad 100644 --- a/pkg/repositories/initialize.go +++ b/pkg/repositories/initialize.go @@ -2,89 +2,31 @@ package repositories import ( "context" - "errors" - "reflect" + "fmt" - errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" + "github.com/flyteorg/datacatalog/pkg/repositories/config" "github.com/flyteorg/datacatalog/pkg/runtime" "github.com/flyteorg/flytestdlib/logger" - "github.com/flyteorg/flytestdlib/promutils" - "github.com/jackc/pgconn" + "github.com/go-gormigrate/gormigrate/v2" ) -var migrationsScope = promutils.NewScope("migrations") -var migrateScope = migrationsScope.NewSubScope("migrate") - -// all postgres servers come by default with a db name named postgres -const defaultDB = "postgres" -const pqInvalidDBCode = "3D000" -const pqDbAlreadyExistsCode = "42P04" - // Migrate This command will run all the migrations for the database func Migrate(ctx context.Context) error { configProvider := runtime.NewConfigurationProvider() dbConfigValues := *configProvider.ApplicationConfiguration().GetDbConfig() - dbName := dbConfigValues.Postgres.DbName - dbHandle, err := NewDBHandle(ctx, dbConfigValues, migrateScope) - + dbHandle, err := NewDBHandle(ctx, dbConfigValues, nil) if err != nil { - // if db does not exist, try creating it - cErr, ok := err.(errors2.ConnectError) - if !ok { - logger.Errorf(ctx, "Failed to cast error of type: %v, err: %v", reflect.TypeOf(err), - err) - panic(err) - } - pqError := cErr.Unwrap().(*pgconn.PgError) - if pqError.Code == pqInvalidDBCode { - logger.Warningf(ctx, "Database [%v] does not exist, trying to create it now", dbName) - - dbConfigValues.Postgres.DbName = defaultDB - setupDBHandler, err := NewDBHandle(ctx, dbConfigValues, migrateScope) - if err != nil { - logger.Errorf(ctx, "Failed to connect to default DB %v, err %v", defaultDB, err) - panic(err) - } - - // Create the database if it doesn't exist - // NOTE: this is non-destructive - if for some reason one does exist an err will be thrown - err = setupDBHandler.CreateDB(dbName) - if err != nil { - logger.Errorf(ctx, "Failed to create DB %v err %v", dbName, err) - panic(err) - } - - dbConfigValues.Postgres.DbName = dbName - dbHandle, err = NewDBHandle(ctx, dbConfigValues, migrateScope) - if err != nil { - logger.Errorf(ctx, "Failed to connect DB err %v", err) - panic(err) - } - } else { - logger.Errorf(ctx, "Failed to connect DB err %v", err) - panic(err) - } + return err } logger.Infof(ctx, "Created DB connection.") - // TODO: checkpoints for migrations - if err := dbHandle.Migrate(ctx); err != nil { - logger.Errorf(ctx, "Failed to migrate. err: %v", err) - panic(err) + m := gormigrate.New(dbHandle.db, gormigrate.DefaultOptions, config.ListMigrations(dbHandle.db)) + if err := m.Migrate(); err != nil { + return fmt.Errorf("database migration failed: %v", err) } - logger.Infof(ctx, "Ran DB migration successfully.") - return nil -} + logger.Infof(ctx, "Migration ran successfully") -func isPgErrorWithCode(err error, code string) bool { - pgErr := &pgconn.PgError{} - if !errors.As(err, &pgErr) { - // err chain does not contain a pgconn.PgError - return false - } - - // pgconn.PgError found in chain and set to code specified - return pgErr.Code == code + return nil } diff --git a/pkg/repositories/models/artifact.go b/pkg/repositories/models/artifact.go index 4b7fa660..d9d89ddc 100644 --- a/pkg/repositories/models/artifact.go +++ b/pkg/repositories/models/artifact.go @@ -11,7 +11,7 @@ type ArtifactKey struct { type Artifact struct { BaseModel ArtifactKey - DatasetUUID string `gorm:"type:uuid;index:artifacts_dataset_uuid_idx"` + DatasetUUID UUIDString `gorm:"index:artifacts_dataset_uuid_idx"` Dataset Dataset `gorm:"association_autocreate:false"` ArtifactData []ArtifactData `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` Partitions []Partition `gorm:"references:ArtifactID;foreignkey:ArtifactID"` diff --git a/pkg/repositories/models/dataset.go b/pkg/repositories/models/dataset.go index 93dcd967..60438557 100644 --- a/pkg/repositories/models/dataset.go +++ b/pkg/repositories/models/dataset.go @@ -1,16 +1,42 @@ package models import ( + "fmt" + "github.com/gofrs/uuid" + + "database/sql/driver" + "gorm.io/gorm" + "gorm.io/gorm/schema" ) +type UUIDString string + +// As per https://gorm.io/docs/data_types.html, custom data types need to implement the following interfaces +func (s *UUIDString) Scan(value interface{}) error { + switch src := value.(type) { + case string: + *s = UUIDString(src) + return nil + case []byte: + *s = UUIDString(src[:]) + return nil + default: + return fmt.Errorf("failed to scan UUID. Type = %v", src) + } +} + +func (s UUIDString) Value() (driver.Value, error) { + return string(s), nil +} + type DatasetKey struct { - Project string `gorm:"primary_key;"` // part of pkey, no index needed as it is first column in the pkey - Name string `gorm:"primary_key;index:dataset_name_idx"` // part of pkey and has separate index for filtering - Domain string `gorm:"primary_key;index:dataset_domain_idx"` // part of pkey and has separate index for filtering - Version string `gorm:"primary_key;index:dataset_version_idx"` // part of pkey and has separate index for filtering - UUID string `gorm:"type:uuid;unique;"` + Project string `gorm:"primary_key;"` // part of pkey, no index needed as it is first column in the pkey + Name string `gorm:"primary_key;index:dataset_name_idx"` // part of pkey and has separate index for filtering + Domain string `gorm:"primary_key;index:dataset_domain_idx"` // part of pkey and has separate index for filtering + Version string `gorm:"primary_key;index:dataset_version_idx"` // part of pkey and has separate index for filtering + UUID UUIDString `gorm:"unique;type:uuid"` } type Dataset struct { @@ -22,7 +48,7 @@ type Dataset struct { type PartitionKey struct { BaseModel - DatasetUUID string `gorm:"type:uuid;primary_key"` + DatasetUUID UUIDString `gorm:"type:uuid;primary_key"` Name string `gorm:"primary_key"` } @@ -33,8 +59,16 @@ func (dataset *Dataset) BeforeCreate(tx *gorm.DB) error { if err != nil { return err } - tx.Model(dataset).Update("UUID", generated) } return nil } + +func (dataset UUIDString) GormDBDataType(db *gorm.DB, field *schema.Field) string { + // use field.Tag, field.TagSettings gets field's tags + // checkout https://github.com/go-gorm/gorm/blob/master/schema/field.go for all options + if db.Dialector.Name() == "mysql" { + return "varchar(36)" + } + return "uuid" +} diff --git a/pkg/repositories/models/partition.go b/pkg/repositories/models/partition.go index 471f2b01..fb77fd5d 100644 --- a/pkg/repositories/models/partition.go +++ b/pkg/repositories/models/partition.go @@ -5,7 +5,7 @@ package models // 2. Get the artifact that has the partitions (x,y,z + tag_name) = latest [x] type Partition struct { BaseModel - DatasetUUID string `gorm:"primary_key;type:uuid"` + DatasetUUID UUIDString `gorm:"primary_key;type:uuid"` Key string `gorm:"primary_key"` Value string `gorm:"primary_key"` ArtifactID string `gorm:"primary_key;index"` // index for JOINs with the Tag/Labels table when querying artifacts diff --git a/pkg/repositories/models/reservation.go b/pkg/repositories/models/reservation.go index 366801ff..414ba724 100644 --- a/pkg/repositories/models/reservation.go +++ b/pkg/repositories/models/reservation.go @@ -4,11 +4,11 @@ import "time" // ReservationKey uniquely identifies a reservation type ReservationKey struct { - DatasetProject string `gorm:"primary_key"` - DatasetName string `gorm:"primary_key"` - DatasetDomain string `gorm:"primary_key"` - DatasetVersion string `gorm:"primary_key"` - TagName string `gorm:"primary_key"` + DatasetProject string `gorm:"size:100;primary_key"` + DatasetName string `gorm:"size:100;primary_key"` + DatasetDomain string `gorm:"size:100;primary_key"` + DatasetVersion string `gorm:"size:100;primary_key"` + TagName string `gorm:"size:100;primary_key"` } // Reservation tracks the metadata needed to allow @@ -18,7 +18,7 @@ type Reservation struct { ReservationKey // Identifies who owns the reservation - OwnerID string + OwnerID string `gorm:"size:100"` // When the reservation will expire ExpiresAt time.Time diff --git a/pkg/repositories/models/tag.go b/pkg/repositories/models/tag.go index 7b239cb3..139d156a 100644 --- a/pkg/repositories/models/tag.go +++ b/pkg/repositories/models/tag.go @@ -1,17 +1,17 @@ package models type TagKey struct { - DatasetProject string `gorm:"primary_key"` - DatasetName string `gorm:"primary_key"` - DatasetDomain string `gorm:"primary_key"` - DatasetVersion string `gorm:"primary_key"` - TagName string `gorm:"primary_key"` + DatasetProject string `gorm:"size:128;primary_key"` + DatasetName string `gorm:"size:128;primary_key"` + DatasetDomain string `gorm:"size:128;primary_key"` + DatasetVersion string `gorm:"size:128;primary_key"` + TagName string `gorm:"size:128;primary_key"` } type Tag struct { BaseModel TagKey ArtifactID string - DatasetUUID string `gorm:"type:uuid;index:tags_dataset_uuid_idx"` - Artifact Artifact `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` + DatasetUUID UUIDString `gorm:"index:tags_dataset_uuid_idx"` + Artifact Artifact `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` } diff --git a/pkg/repositories/mysql_repo.go b/pkg/repositories/mysql_repo.go new file mode 100644 index 00000000..6374098b --- /dev/null +++ b/pkg/repositories/mysql_repo.go @@ -0,0 +1,41 @@ +package repositories + +import ( + "github.com/flyteorg/datacatalog/pkg/repositories/errors" + "github.com/flyteorg/datacatalog/pkg/repositories/gormimpl" + "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" + "github.com/flyteorg/flytestdlib/promutils" + "gorm.io/gorm" +) + +type MySqlRepo struct { + datasetRepo interfaces.DatasetRepo + artifactRepo interfaces.ArtifactRepo + tagRepo interfaces.TagRepo + reservationRepo interfaces.ReservationRepo +} + +func (dc *MySqlRepo) DatasetRepo() interfaces.DatasetRepo { + return dc.datasetRepo +} + +func (dc *MySqlRepo) ArtifactRepo() interfaces.ArtifactRepo { + return dc.artifactRepo +} + +func (dc *MySqlRepo) TagRepo() interfaces.TagRepo { + return dc.tagRepo +} + +func (dc *MySqlRepo) ReservationRepo() interfaces.ReservationRepo { + return dc.reservationRepo +} + +func NewMySqlRepo(db *gorm.DB, errorTransformer errors.ErrorTransformer, scope promutils.Scope) interfaces.DataCatalogRepo { + return &MySqlRepo{ + datasetRepo: gormimpl.NewDatasetRepo(db, errorTransformer, scope.NewSubScope("dataset")), + artifactRepo: gormimpl.NewArtifactRepo(db, errorTransformer, scope.NewSubScope("artifact")), + tagRepo: gormimpl.NewTagRepo(db, errorTransformer, scope.NewSubScope("tag")), + reservationRepo: gormimpl.NewReservationRepo(db, errorTransformer, scope.NewSubScope("reservation")), + } +} diff --git a/pkg/repositories/postgres_repo.go b/pkg/repositories/postgres_repo.go index c51d5db6..5c2f0882 100644 --- a/pkg/repositories/postgres_repo.go +++ b/pkg/repositories/postgres_repo.go @@ -31,6 +31,8 @@ func (dc *PostgresRepo) ReservationRepo() interfaces.ReservationRepo { return dc.reservationRepo } +// NewPostgresRepo - The only thing postgres about this is the fact that it's usually called with an error transformer +// that understands postgres error codes func NewPostgresRepo(db *gorm.DB, errorTransformer errors.ErrorTransformer, scope promutils.Scope) interfaces.DataCatalogRepo { return &PostgresRepo{ datasetRepo: gormimpl.NewDatasetRepo(db, errorTransformer, scope.NewSubScope("dataset")), diff --git a/pkg/repositories/transformers/artifact.go b/pkg/repositories/transformers/artifact.go index ca707dfb..724db898 100644 --- a/pkg/repositories/transformers/artifact.go +++ b/pkg/repositories/transformers/artifact.go @@ -46,7 +46,7 @@ func FromArtifactModel(artifact models.Artifact) (*datacatalog.Artifact, error) Domain: artifact.DatasetDomain, Name: artifact.DatasetName, Version: artifact.DatasetVersion, - UUID: artifact.DatasetUUID, + UUID: string(artifact.DatasetUUID), } metadata, err := unmarshalMetadata(artifact.SerializedMetadata) diff --git a/pkg/repositories/transformers/dataset.go b/pkg/repositories/transformers/dataset.go index f0d6eebf..4a38b5f1 100644 --- a/pkg/repositories/transformers/dataset.go +++ b/pkg/repositories/transformers/dataset.go @@ -26,7 +26,7 @@ func CreateDatasetModel(dataset *datacatalog.Dataset) (*models.Dataset, error) { Domain: dataset.Id.Domain, Name: dataset.Id.Name, Version: dataset.Id.Version, - UUID: dataset.Id.UUID, + UUID: models.UUIDString(dataset.Id.UUID), }, SerializedMetadata: serializedMetadata, PartitionKeys: partitionKeys, @@ -40,7 +40,7 @@ func FromDatasetID(datasetID *datacatalog.DatasetID) models.DatasetKey { Domain: datasetID.Domain, Name: datasetID.Name, Version: datasetID.Version, - UUID: datasetID.UUID, + UUID: models.UUIDString(datasetID.UUID), } } @@ -54,7 +54,7 @@ func FromDatasetModel(dataset models.Dataset) (*datacatalog.Dataset, error) { partitionKeyStrings := FromPartitionKeyModel(dataset.PartitionKeys) return &datacatalog.Dataset{ Id: &datacatalog.DatasetID{ - UUID: dataset.UUID, + UUID: string(dataset.UUID), Project: dataset.Project, Domain: dataset.Domain, Name: dataset.Name, diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 0591b7da..702fcfd7 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -103,7 +103,7 @@ func NewDataCatalogService() *DataCatalogService { } dbConfigValues := configProvider.ApplicationConfiguration().GetDbConfig() - repos := repositories.GetRepository(ctx, repositories.POSTGRES, *dbConfigValues, catalogScope) + repos := repositories.GetRepository(ctx, *dbConfigValues, catalogScope) logger.Infof(ctx, "Created DB connection.") // Serve profiling endpoint.