diff --git a/go.mod b/go.mod index ab32a829..08fac947 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,9 @@ require ( cloud.google.com/go/storage v1.29.0 github.com/AlecAivazis/survey/v2 v2.3.6 github.com/ClickHouse/clickhouse-go v1.4.5 + github.com/IBM/sarama v1.43.2 github.com/MakeNowJust/heredoc v1.0.0 + github.com/aliyun/aliyun-odps-go-sdk v0.3.14 github.com/aws/aws-sdk-go v1.44.151 github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba github.com/cenkalti/backoff/v4 v4.2.1 @@ -38,6 +40,7 @@ require ( github.com/ory/dockertest/v3 v3.9.1 github.com/pkg/errors v0.9.1 github.com/prestodb/presto-go-client v0.0.0-20211201125635-ad28cec17d6c + github.com/rs/zerolog v1.32.0 github.com/schollz/progressbar/v3 v3.13.1 github.com/segmentio/kafka-go v0.4.47 github.com/sijms/go-ora/v2 v2.7.9 @@ -64,7 +67,7 @@ require ( google.golang.org/api v0.114.0 google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea google.golang.org/grpc v1.55.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -77,10 +80,12 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-storage-blob-go v0.14.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/IBM/sarama v1.43.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/alecthomas/chroma v0.10.0 // indirect + github.com/alibabacloud-go/debug v1.0.1 // indirect + github.com/alibabacloud-go/tea v1.2.2 // indirect + github.com/aliyun/credentials-go v1.3.10 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect github.com/apache/arrow/go/v11 v11.0.0 // indirect @@ -136,11 +141,12 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect @@ -158,6 +164,7 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jeremywohl/flatten v1.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v0.0.0-20170703010042-c7d06af17c68 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect @@ -178,6 +185,8 @@ require ( github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.13.0 // indirect @@ -198,7 +207,6 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.4.3 // indirect - github.com/rs/zerolog v1.32.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil/v3 v3.23.4 // indirect github.com/shoenig/go-m1cpu v0.1.5 // indirect @@ -232,7 +240,7 @@ require ( go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.22.0 // indirect - golang.org/x/mod v0.9.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/term v0.19.0 // indirect diff --git a/go.sum b/go.sum index 43dd4e24..affa47aa 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,7 @@ cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31 cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/AlecAivazis/survey/v2 v2.3.5/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI= github.com/AlecAivazis/survey/v2 v2.3.6 h1:NvTuVHISgTHEHeBFqt6BHOe4Ny/NwGZr7w+F8S9ziyw= @@ -169,7 +170,10 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= github.com/alecthomas/chroma v0.8.2/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= @@ -184,6 +188,15 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= +github.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= +github.com/alibabacloud-go/debug v1.0.1 h1:MsW9SmUtbb1Fnt3ieC6NNZi6aEwrXfDksD4QA6GSbPg= +github.com/alibabacloud-go/debug v1.0.1/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= +github.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU= +github.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk= +github.com/aliyun/aliyun-odps-go-sdk v0.3.14 h1:H+p9+2n4kSBOiJvndKC7M+0UxK0mW4azQHQzHy2hI2A= +github.com/aliyun/aliyun-odps-go-sdk v0.3.14/go.mod h1:t/tgF/iN5aAs/gLL7sEI8/qdax4NuFCKEjO3OJbHZqI= +github.com/aliyun/credentials-go v1.3.10 h1:45Xxrae/evfzQL9V10zL3xX31eqgLWEaIdCoPipOEQA= +github.com/aliyun/credentials-go v1.3.10/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -295,6 +308,7 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/briandowns/spinner v1.18.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/briandowns/spinner v1.20.0 h1:GQq1Yf1KyzYT8CY19GzWrDKP6hYOFB6J72Ks7d8aO1U= github.com/briandowns/spinner v1.20.0/go.mod h1:TcwZHb7Wb6vn/+bcVv1UXEzaA4pLS7yznHlkY/HzH44= @@ -582,6 +596,7 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -599,7 +614,10 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/latin-modern v0.3.0/go.mod h1:ysEQXnuT/sCDOAONxC7ImeEDVINbltClhasMAqEtRK0= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.3.0/go.mod h1:jdJ+cqF+F4SUL2V+qxBth8fvBpBDS7yloUL5Fi8GTGY= github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -616,6 +634,8 @@ github.com/go-kivik/kivik v2.0.0+incompatible/go.mod h1:nIuJ8z4ikBrVUSk3Ua8NoDqY github.com/go-kivik/kiviktest v2.0.0+incompatible h1:y1RyPHqWQr+eFlevD30Tr3ipiPCxK78vRoD3o9YysjI= github.com/go-kivik/kiviktest v2.0.0+incompatible/go.mod h1:JdhVyzixoYhoIDUt6hRf1yAfYyaDa5/u9SDOindDkfQ= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9/go.mod h1:gWuR/CrFDDeVRFQwHPvsv9soJVB/iqymhuZQuJ3a9OM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -647,6 +667,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -767,8 +789,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= -github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -843,8 +865,9 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -983,6 +1006,7 @@ github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= @@ -1012,6 +1036,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -1188,9 +1213,11 @@ github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7ID github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= @@ -1296,12 +1323,14 @@ github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.11/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= @@ -1393,6 +1422,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= @@ -1494,6 +1524,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -1715,8 +1746,10 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1733,7 +1766,9 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1743,6 +1778,11 @@ golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1768,9 +1808,11 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1854,8 +1896,10 @@ golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2057,6 +2101,7 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2070,6 +2115,7 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2085,6 +2131,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -2179,6 +2226,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= @@ -2196,10 +2244,12 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNq gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -2411,8 +2461,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2484,6 +2534,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= @@ -2567,5 +2618,3 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -source.golabs.io/cloud-platform/kafka-security-go v0.1.2 h1:94/wLnbq/3anb7sz4vgKyT5lBPtg78iRO9kdO7zbNHY= -source.golabs.io/cloud-platform/kafka-security-go v0.1.2/go.mod h1:FNvdKrqx9Uey0qpULdbQfpCc6T1Zqgta6okMrQvss8A= diff --git a/plugins/extractors/maxcompute/README.md b/plugins/extractors/maxcompute/README.md new file mode 100644 index 00000000..322a7c0d --- /dev/null +++ b/plugins/extractors/maxcompute/README.md @@ -0,0 +1,90 @@ +# maxcompute +## Usage +The `maxcompute` extractor allows you to extract metadata from MaxCompute tables and schemas. +It supports configuration for project name, endpoint, access keys, schema name, exclusions, and concurrency. + +```yaml +source: + name: maxcompute + config: + project_name: goto_test + endpoint_project: http://goto_test-maxcompute.com + access_key: + id: access_key_id + secret: access_key_secret + schema_name: DEFAULT + exclude: + schemas: + - schema_a + - schema_b + tables: + - schema_c.table_a + concurrency: 10 +``` + +## Inputs + +| Key | Value | Example | Description | | +| :-- | :---- | :------ | :---------- | :-- | +| `project_name` | `string` | `goto_test` | MaxCompute Project Name | *required* | +| `endpoint_project` | `string` | `http://goto_test-maxcompute.com` | Endpoint Project URL | *required* | +| `access_key.id` | `string` | `access_key_id` | Access Key ID | *required* | +| `access_key.secret` | `string` | `access_key_secret` | Access Key Secret | *required* | +| `schema_name` | `string` | `DEFAULT` | Default schema name | *optional* | +| `exclude.schemas` | `[]string` | `["schema_a", "schema_b"]` | List of schemas to exclude | *optional* | +| `exclude.tables` | `[]string` | `["schema_c.table_a"]` | List of tables to exclude | *optional* | +| `concurrency` | `int` | `10` | Number of concurrent requests to MaxCompute | *optional* | + +### *Notes* + +- Leaving `access_key` blank will default to [MaxCompute's default authentication][maxcompute-default-auth]. + +## Outputs + +| Field | Sample Value | Description | +|:-------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------| +| `resource.urn` | `project_name.schema_name.table_name` | | +| `resource.name` | `table_name` | | +| `resource.service` | `maxcompute` | | +| `description` | `table description` | | +| `schema` | [][Column](#column) | | +| `properties.partition_data` | `"partition_data": {"partition_field": "data_date", "require_partition_filter": false, "time_partition": {"partition_by": "DAY","partition_expire": 0 } }` | partition related data for time and range partitioning. | +| `properties.partition_field` | `created_at` | returns the field on which table is time partitioned | + +### Partition Data + +| Field | Sample Value | Description | +|:------------------------------------------|:-------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `partition_field` | `created_at` | field on which the table is partitioned either by TimePartitioning or RangePartitioning. In case field is empty for TimePartitioning _PARTITIONTIME is returned instead of empty. | +| `require_partition_filter` | `true` | boolean value which denotes if every query on the MaxCompute table must include at least one predicate that only references the partitioning column | +| `time_partition.partition_by` | `HOUR` | returns partition type HOUR/DAY/MONTH/YEAR | +| `time_partition.partition_expire_seconds` | `0` | time in which data will expire from this partition. If 0 it will not expire. | +| `range_partition.interval` | `10` | width of a interval range | +| `range_partition.start` | `0` | start value for partition inclusive of this value | +| `range_partition.end` | `100` | end value for partition exclusive of this value | + + +### Column + +| Field | Sample Value | +|:--------------|:---------------------------------------| +| `name` | `total_price` | +| `description` | `item's total price` | +| `data_type` | `decimal` | +| `is_nullable` | `true` | + +### Join + +| Field | Sample Value | +|:-------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------| +| `urn` | `project_name.schema_name.table_name` | +| `count` | `3` | +| `conditions` | [`"ON target.column_1 = source.column_1 and target.param_name = source.param_name"`,`"ON DATE(target.event_timestamp) = DATE(source.event_timestamp)"`] | + +## Contributing + +Refer to the [contribution guidelines](../../../docs/docs/contribute/guide.md#adding-a-new-extractor) for information on +contributing to this module. + +[maxcompute-default-auth]: https://www.alibabacloud.com/help/doc-detail/27800.htm + diff --git a/plugins/extractors/maxcompute/client/client.go b/plugins/extractors/maxcompute/client/client.go new file mode 100644 index 00000000..35a980ed --- /dev/null +++ b/plugins/extractors/maxcompute/client/client.go @@ -0,0 +1,73 @@ +package client + +import ( + "context" + + "github.com/aliyun/aliyun-odps-go-sdk/odps" + "github.com/aliyun/aliyun-odps-go-sdk/odps/account" + "github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema" + "github.com/goto/meteor/plugins" + "github.com/goto/meteor/plugins/extractors/maxcompute/config" + "github.com/goto/salt/log" +) + +type Client struct { + client *odps.Odps + project *odps.Project + log log.Logger +} + +func New(conf config.Config) *Client { + aliAccount := account.NewAliyunAccount(conf.AccessKey.ID, conf.AccessKey.Secret) + client := odps.NewOdps(aliAccount, conf.EndpointProject) + client.SetDefaultProjectName(conf.ProjectName) + + project := client.Project(conf.ProjectName) + + return &Client{ + client: client, + project: project, + log: plugins.GetLog(), + } +} + +func (c *Client) ListSchema(context.Context) (schemas []*odps.Schema, err error) { + err = c.project.Schemas().List(func(schema *odps.Schema, err2 error) { + if err2 != nil { + err = err2 + c.log.Error("failed to process schema", "with error:", err) + return + } + schemas = append(schemas, schema) + }) + + return schemas, err +} + +func (c *Client) ListTable(_ context.Context, schemaName string) (tables []*odps.Table, err error) { + t := odps.NewTables(c.client, c.project.Name(), schemaName) + t.List( + func(table *odps.Table, err2 error) { + if err2 != nil { + err = err2 + c.log.Error("failed to process table", "with error:", err) + return + } + tables = append(tables, table) + }, + ) + return tables, err +} + +func (*Client) GetTableSchema(_ context.Context, table *odps.Table) (string, *tableschema.TableSchema, error) { + err := table.Load() + tableSchema := table.Schema() + if err != nil { + isView := tableSchema.IsVirtualView || tableSchema.IsMaterializedView + isLoaded := table.IsLoaded() + if !isView || (isView && !isLoaded) { + return "", nil, err + } + } + return table.Type().String(), &tableSchema, nil +} diff --git a/plugins/extractors/maxcompute/config/config.go b/plugins/extractors/maxcompute/config/config.go new file mode 100644 index 00000000..a0f17a5a --- /dev/null +++ b/plugins/extractors/maxcompute/config/config.go @@ -0,0 +1,16 @@ +package config + +type Config struct { + ProjectName string `mapstructure:"project_name"` + EndpointProject string `mapstructure:"endpoint_project"` + AccessKey struct { + ID string `mapstructure:"id"` + Secret string `mapstructure:"secret"` + } `mapstructure:"access_key"` + SchemaName string `mapstructure:"schema_name,omitempty"` + Exclude struct { + Schemas []string `mapstructure:"schemas"` + Tables []string `mapstructure:"tables"` + } `mapstructure:"exclude,omitempty"` + Concurrency int `mapstructure:"concurrency,omitempty"` +} diff --git a/plugins/extractors/maxcompute/maxcompute.go b/plugins/extractors/maxcompute/maxcompute.go new file mode 100644 index 00000000..7741020b --- /dev/null +++ b/plugins/extractors/maxcompute/maxcompute.go @@ -0,0 +1,310 @@ +package maxcompute + +import ( + "context" + _ "embed" // used to print the embedded assets + "fmt" + "time" + + "github.com/aliyun/aliyun-odps-go-sdk/odps" + "github.com/aliyun/aliyun-odps-go-sdk/odps/common" + "github.com/aliyun/aliyun-odps-go-sdk/odps/datatype" + "github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema" + "github.com/goto/meteor/models" + v1beta2 "github.com/goto/meteor/models/gotocompany/assets/v1beta2" + "github.com/goto/meteor/plugins" + "github.com/goto/meteor/plugins/extractors/maxcompute/client" + "github.com/goto/meteor/plugins/extractors/maxcompute/config" + "github.com/goto/meteor/registry" + "github.com/goto/meteor/utils" + "github.com/goto/salt/log" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type Extractor struct { + plugins.BaseExtractor + logger log.Logger + config config.Config + + client Client + newClient NewClientFunc + eg *errgroup.Group +} + +type NewClientFunc func(ctx context.Context, logger log.Logger, conf config.Config) (Client, error) + +//go:embed README.md +var summary string + +var sampleConfig = ` +project_name: goto_test +endpoint_project: https://service.ap-southeast-5.maxcompute.aliyun.com/api +access_key: + id: access_key_id + secret: access_key_secret +schema_name: default +exclude: + schemas: + - schema_a + - schema_b + tables: + - schema_c.table_a +concurrency: 10 +` + +var info = plugins.Info{ + Description: "MaxCompute metadata and metrics", + SampleConfig: sampleConfig, + Tags: []string{"alicloud", "table"}, + Summary: summary, +} + +//go:generate mockery --name=Client -r --case underscore --with-expecter --structname MaxComputeClient --filename maxcompute_client_mock.go --output=./mocks +type Client interface { + ListSchema(ctx context.Context) ([]*odps.Schema, error) + ListTable(ctx context.Context, schemaName string) ([]*odps.Table, error) + GetTableSchema(ctx context.Context, table *odps.Table) (string, *tableschema.TableSchema, error) +} + +func New(logger log.Logger, clientFunc NewClientFunc) *Extractor { + e := &Extractor{ + logger: logger, + newClient: clientFunc, + } + e.BaseExtractor = plugins.NewBaseExtractor(info, &e.config) + e.ScopeNotRequired = true + + return e +} + +func (e *Extractor) Init(ctx context.Context, conf plugins.Config) error { + if err := e.BaseExtractor.Init(ctx, conf); err != nil { + return err + } + + if e.config.ProjectName == "" { + return fmt.Errorf("project_name is required") + } + if e.config.AccessKey.ID == "" || e.config.AccessKey.Secret == "" { + return fmt.Errorf("access_key is required") + } + if e.config.EndpointProject == "" { + return fmt.Errorf("endpoint_project is required") + } + if e.config.Concurrency == 0 { + e.config.Concurrency = 1 + } + + var err error + e.client, err = e.newClient(ctx, e.logger, e.config) + if err != nil { + return err + } + + e.eg = &errgroup.Group{} + e.eg.SetLimit(e.config.Concurrency) + + return nil +} + +func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) error { + schemas, err := e.client.ListSchema(ctx) + if err != nil && len(schemas) == 0 { + return err + } + + for _, schema := range schemas { + if e.config.SchemaName != "" && schema.Name() != e.config.SchemaName { + continue + } + if contains(e.config.Exclude.Schemas, schema.Name()) { + continue + } + + err := e.fetchTablesFromSchema(ctx, schema, emit) + if err != nil { + return err + } + } + + return e.eg.Wait() +} + +func (e *Extractor) fetchTablesFromSchema(ctx context.Context, schema *odps.Schema, emit plugins.Emit) error { + tables, err := e.client.ListTable(ctx, schema.Name()) + if err != nil && len(tables) == 0 { + return err + } + + for _, table := range tables { + if contains(e.config.Exclude.Tables, fmt.Sprintf("%s.%s", table.SchemaName(), table.Name())) { + continue + } + + tbl := table + e.eg.Go(func() error { + return e.processTable(ctx, schema, tbl, emit) + }) + } + + return nil +} + +func (e *Extractor) processTable(ctx context.Context, schema *odps.Schema, table *odps.Table, emit plugins.Emit) error { + tableType, tableSchema, err := e.client.GetTableSchema(ctx, table) + if err != nil { + return err + } + + asset, err := e.buildAsset(schema, table, tableType, tableSchema) + if err != nil { + e.logger.Error("failed to build asset", "table", table.Name(), "error", err) + return err + } + + emit(models.NewRecord(asset)) + return nil +} + +func (e *Extractor) buildAsset(schema *odps.Schema, _ *odps.Table, tableType string, tableSchema *tableschema.TableSchema) (*v1beta2.Asset, error) { + defaultSchema := "default" + schemaName := schema.Name() + if schemaName == "" { + schemaName = defaultSchema + } + + tableURN := plugins.MaxComputeURN(e.config.ProjectName, schemaName, tableSchema.TableName) + + asset := &v1beta2.Asset{ + Urn: tableURN, + Name: tableSchema.TableName, + Type: tableType, + Description: tableSchema.Comment, + CreateTime: timestamppb.New(time.Time(tableSchema.CreateTime)), + UpdateTime: timestamppb.New(time.Time(tableSchema.LastModifiedTime)), + Service: "maxcompute", + } + + tableAttributesData := e.buildTableAttributesData(schemaName, tableSchema) + + var columns []*v1beta2.Column + for i, col := range tableSchema.Columns { + columnData := &v1beta2.Column{ + Name: col.Name, + DataType: dataTypeToString(col.Type), + Description: col.Comment, + IsNullable: col.IsNullable, + Attributes: utils.TryParseMapToProto(buildColumnAttributesData(&tableSchema.Columns[i])), + Columns: buildColumns(col.Type), + } + columns = append(columns, columnData) + } + + tableData := &v1beta2.Table{ + Attributes: utils.TryParseMapToProto(tableAttributesData), + Columns: columns, + CreateTime: timestamppb.New(time.Time(tableSchema.CreateTime)), + UpdateTime: timestamppb.New(time.Time(tableSchema.LastModifiedTime)), + } + + tbl, err := anypb.New(tableData) + if err != nil { + e.logger.Warn("error creating Any struct", "error", err) + } + asset.Data = tbl + + return asset, nil +} + +func buildColumns(dataType datatype.DataType) []*v1beta2.Column { + if dataType.ID() != datatype.STRUCT { + return nil + } + + structType, ok := dataType.(datatype.StructType) + if !ok { + return nil + } + + var columns []*v1beta2.Column + for _, field := range structType.Fields { + column := &v1beta2.Column{ + Name: field.Name, + DataType: dataTypeToString(field.Type), + Columns: buildColumns(field.Type), + } + columns = append(columns, column) + } + return columns +} + +func (e *Extractor) buildTableAttributesData(schemaName string, tableInfo *tableschema.TableSchema) map[string]interface{} { + attributesData := map[string]interface{}{} + + attributesData["project_name"] = e.config.ProjectName + attributesData["schema"] = schemaName + + rb := common.ResourceBuilder{ProjectName: e.config.ProjectName} + attributesData["resource_url"] = rb.Table(tableInfo.TableName) + + if tableInfo.ViewText != "" { + attributesData["sql"] = tableInfo.ViewText + } + + if tableInfo.PartitionColumns != nil && len(tableInfo.PartitionColumns) > 0 { + partitionNames := make([]string, len(tableInfo.PartitionColumns)) + for i, column := range tableInfo.PartitionColumns { + partitionNames[i] = column.Name + } + attributesData["partition_fields"] = partitionNames + } + + return attributesData +} + +func buildColumnAttributesData(column *tableschema.Column) map[string]interface{} { + attributesData := map[string]interface{}{} + + if column == nil { + return attributesData + } + + if column.Label != "" { + attributesData["label"] = column.Label + } + + return attributesData +} + +func dataTypeToString(dataType datatype.DataType) string { + if dataType.ID() == datatype.MAP { + return dataType.Name() + } + if dataType.ID() == datatype.ARRAY { + return dataType.Name() + } + return dataType.ID().String() +} + +func contains(slice []string, item string) bool { + for _, s := range slice { + if s == item { + return true + } + } + return false +} + +func init() { + if err := registry.Extractors.Register("maxcompute", func() plugins.Extractor { + return New(plugins.GetLog(), CreateClient) + }); err != nil { + panic(err) + } +} + +func CreateClient(_ context.Context, _ log.Logger, conf config.Config) (Client, error) { + return client.New(conf), nil +} diff --git a/plugins/extractors/maxcompute/maxcompute_test.go b/plugins/extractors/maxcompute/maxcompute_test.go new file mode 100644 index 00000000..3caa88f1 --- /dev/null +++ b/plugins/extractors/maxcompute/maxcompute_test.go @@ -0,0 +1,239 @@ +//go:build plugins +// +build plugins + +package maxcompute_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/aliyun/aliyun-odps-go-sdk/odps" + "github.com/aliyun/aliyun-odps-go-sdk/odps/common" + "github.com/aliyun/aliyun-odps-go-sdk/odps/datatype" + "github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema" + v1beta2 "github.com/goto/meteor/models/gotocompany/assets/v1beta2" + "github.com/goto/meteor/plugins" + "github.com/goto/meteor/plugins/extractors/maxcompute" + "github.com/goto/meteor/plugins/extractors/maxcompute/config" + "github.com/goto/meteor/plugins/extractors/maxcompute/mocks" + mocks2 "github.com/goto/meteor/test/mocks" + "github.com/goto/meteor/test/utils" + "github.com/goto/salt/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +const ( + projectID = "test-project-id" +) + +func TestInit(t *testing.T) { + t.Run("should return error if config is invalid", func(t *testing.T) { + extr := maxcompute.New(utils.Logger, maxcompute.CreateClient) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := extr.Init(ctx, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": "", + }, + }) + + assert.ErrorContains(t, err, "project_name is required") + }) + + t.Run("should return no error", func(t *testing.T) { + extr := maxcompute.New(utils.Logger, maxcompute.CreateClient) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := extr.Init(ctx, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": projectID, + "access_key": map[string]interface{}{ + "id": "access_key_id", + "secret": "access_key_secret", + }, + "endpoint_project": "https://example.com/some-api", + }, + }) + + assert.NoError(t, err) + }) +} + +func createClient(client *mocks.MaxComputeClient) func(ctx context.Context, logger log.Logger, config config.Config) (maxcompute.Client, error) { + return func(ctx context.Context, logger log.Logger, config config.Config) (maxcompute.Client, error) { + return client, nil + } +} + +func TestExtract(t *testing.T) { + schema1 := []*odps.Schema{ + odps.NewSchema(nil, projectID, "my_schema"), + } + + table1 := []*odps.Table{ + odps.NewTable(nil, projectID, "my_schema", "dummy_table"), + odps.NewTable(nil, projectID, "my_schema", "new_table"), + } + + c1 := tableschema.Column{ + Name: "id", + Type: datatype.BigIntType, + IsNullable: true, + } + + c2 := tableschema.Column{ + Name: "name", + Type: datatype.StructType{ + Fields: []datatype.StructFieldType{ + { + Name: "first_name", + Type: datatype.StringType, + }, + { + Name: "last_name", + Type: datatype.StringType, + }, + }, + }, + IsNullable: true, + } + + // Schema for dummy_table + dummyTableSchemaBuilder := tableschema.NewSchemaBuilder() + dummyTableSchemaBuilder.Name("dummy_table"). + Columns(c1, c2) + dummyTableSchema := dummyTableSchemaBuilder.Build() + dummyTableSchema.ViewText = "SELECT id, name, user_info\nFROM test-project-id.default.my_dummy_table" + dummyCreateTime, err := time.Parse(time.RFC3339, "2024-11-14T06:41:35Z") + if err != nil { + t.Fatalf("failed to parse create time for dummy_table: %v", err) + } + dummyTableSchema.CreateTime = common.GMTTime(dummyCreateTime) + dummyTableSchema.LastModifiedTime = common.GMTTime(dummyCreateTime) + dummyTableSchema.Comment = "dummy table description" + + c3 := tableschema.Column{ + Name: "user_id", + Type: datatype.BigIntType, + Comment: "Unique identifier for users", + IsNullable: false, + } + + c4 := tableschema.Column{ + Name: "email", + Type: datatype.StringType, + IsNullable: false, + Comment: "User email address", + } + + // Schema for new_table + newTableSchemaBuilder := tableschema.NewSchemaBuilder() + newTableSchemaBuilder.Name("new_table"). + Columns(c3, c4) + newTableSchema := newTableSchemaBuilder.Build() + newTableSchema.ViewText = "SELECT user_id, email FROM test-project-id.my_schema.new_table" + newCreateTime, err := time.Parse(time.RFC3339, "2024-11-18T08:00:00Z") + if err != nil { + t.Fatalf("failed to parse create time for new_table: %v", err) + } + newTableSchema.CreateTime = common.GMTTime(newCreateTime) + newTableSchema.LastModifiedTime = common.GMTTime(newCreateTime) + + // Schema mapping + schemaMapping := map[string]*tableschema.TableSchema{ + "dummy_table": &dummyTableSchema, + "new_table": &newTableSchema, + } + + runTest := func(t *testing.T, cfg plugins.Config, mockSetup func(mockClient *mocks.MaxComputeClient)) ([]*v1beta2.Asset, error) { + mockClient := mocks.NewMaxComputeClient(t) + if mockSetup != nil { + mockSetup(mockClient) + } + extr := maxcompute.New(utils.Logger, createClient(mockClient)) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + err := extr.Init(ctx, cfg) + if err != nil { + return nil, err + } + + emitter := mocks2.NewEmitter() + err = extr.Extract(ctx, emitter.Push) + + actual := emitter.GetAllData() + return actual, err + } + + t.Run("should return no error", func(t *testing.T) { + actual, err := runTest(t, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": projectID, + "access_key": map[string]interface{}{ + "id": "access_key_id", + "secret": "access_key_secret", + }, + "endpoint_project": "https://example.com/some-api", + }, + }, func(mockClient *mocks.MaxComputeClient) { + mockClient.EXPECT().ListSchema(mock.Anything).Return(schema1, nil) + mockClient.EXPECT().ListTable(mock.Anything, "my_schema").Return(table1, nil) + mockClient.EXPECT().GetTableSchema(mock.Anything, table1[0]).Return("VIRTUAL_VIEW", schemaMapping[table1[0].Name()], nil) + mockClient.EXPECT().ListTable(mock.Anything, "my_schema").Return(table1[1:], nil) + mockClient.EXPECT().GetTableSchema(mock.Anything, table1[1]).Return("MANAGED_TABLE", schemaMapping[table1[1].Name()], nil) + }) + + assert.Nil(t, err) + assert.NotEmpty(t, actual) + utils.AssertProtosWithJSONFile(t, "testdata/expected-assets.json", actual) + }) + + t.Run("should exclude tables based on exclusion rules", func(t *testing.T) { + actual, err := runTest(t, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": projectID, + "access_key": map[string]interface{}{ + "id": "access_key_id", + "secret": "access_key_secret", + }, + "endpoint_project": "https://example.com/some-api", + "exclude": map[string]interface{}{ + "tables": []string{"my_schema.dummy_table"}, + }, + }, + }, func(mockClient *mocks.MaxComputeClient) { + mockClient.EXPECT().ListSchema(mock.Anything).Return(schema1, nil) + mockClient.EXPECT().ListTable(mock.Anything, "my_schema").Return(table1[1:], nil) + mockClient.EXPECT().GetTableSchema(mock.Anything, table1[1]).Return("MANAGED_TABLE", schemaMapping[table1[1].Name()], nil) + }) + + assert.Nil(t, err) + assert.NotEmpty(t, actual) + utils.AssertProtosWithJSONFile(t, "testdata/expected-assets-with-table-exclusion.json", actual) + }) + + t.Run("should return error if ListSchema fails", func(t *testing.T) { + actual, err := runTest(t, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": projectID, + "access_key": map[string]interface{}{ + "id": "access_key_id", + "secret": "access_key_secret", + }, + "endpoint_project": "https://example.com/some-api", + }, + }, func(mockClient *mocks.MaxComputeClient) { + mockClient.EXPECT().ListSchema(mock.Anything).Return(nil, fmt.Errorf("ListSchema fails")) + }) + assert.ErrorContains(t, err, "ListSchema fails") + assert.Nil(t, actual) + }) +} diff --git a/plugins/extractors/maxcompute/mocks/maxcompute_client_mock.go b/plugins/extractors/maxcompute/mocks/maxcompute_client_mock.go new file mode 100644 index 00000000..de4f1dbe --- /dev/null +++ b/plugins/extractors/maxcompute/mocks/maxcompute_client_mock.go @@ -0,0 +1,223 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + odps "github.com/aliyun/aliyun-odps-go-sdk/odps" + + tableschema "github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema" +) + +// MaxComputeClient is an autogenerated mock type for the Client type +type MaxComputeClient struct { + mock.Mock +} + +type MaxComputeClient_Expecter struct { + mock *mock.Mock +} + +func (_m *MaxComputeClient) EXPECT() *MaxComputeClient_Expecter { + return &MaxComputeClient_Expecter{mock: &_m.Mock} +} + +// GetTableSchema provides a mock function with given fields: ctx, table +func (_m *MaxComputeClient) GetTableSchema(ctx context.Context, table *odps.Table) (string, *tableschema.TableSchema, error) { + ret := _m.Called(ctx, table) + + if len(ret) == 0 { + panic("no return value specified for GetTableSchema") + } + + var r0 string + var r1 *tableschema.TableSchema + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, *odps.Table) (string, *tableschema.TableSchema, error)); ok { + return rf(ctx, table) + } + if rf, ok := ret.Get(0).(func(context.Context, *odps.Table) string); ok { + r0 = rf(ctx, table) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, *odps.Table) *tableschema.TableSchema); ok { + r1 = rf(ctx, table) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*tableschema.TableSchema) + } + } + + if rf, ok := ret.Get(2).(func(context.Context, *odps.Table) error); ok { + r2 = rf(ctx, table) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MaxComputeClient_GetTableSchema_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTableSchema' +type MaxComputeClient_GetTableSchema_Call struct { + *mock.Call +} + +// GetTableSchema is a helper method to define mock.On call +// - ctx context.Context +// - table *odps.Table +func (_e *MaxComputeClient_Expecter) GetTableSchema(ctx interface{}, table interface{}) *MaxComputeClient_GetTableSchema_Call { + return &MaxComputeClient_GetTableSchema_Call{Call: _e.mock.On("GetTableSchema", ctx, table)} +} + +func (_c *MaxComputeClient_GetTableSchema_Call) Run(run func(ctx context.Context, table *odps.Table)) *MaxComputeClient_GetTableSchema_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*odps.Table)) + }) + return _c +} + +func (_c *MaxComputeClient_GetTableSchema_Call) Return(_a0 string, _a1 *tableschema.TableSchema, _a2 error) *MaxComputeClient_GetTableSchema_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *MaxComputeClient_GetTableSchema_Call) RunAndReturn(run func(context.Context, *odps.Table) (string, *tableschema.TableSchema, error)) *MaxComputeClient_GetTableSchema_Call { + _c.Call.Return(run) + return _c +} + +// ListSchema provides a mock function with given fields: ctx +func (_m *MaxComputeClient) ListSchema(ctx context.Context) ([]*odps.Schema, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for ListSchema") + } + + var r0 []*odps.Schema + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*odps.Schema, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []*odps.Schema); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*odps.Schema) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MaxComputeClient_ListSchema_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListSchema' +type MaxComputeClient_ListSchema_Call struct { + *mock.Call +} + +// ListSchema is a helper method to define mock.On call +// - ctx context.Context +func (_e *MaxComputeClient_Expecter) ListSchema(ctx interface{}) *MaxComputeClient_ListSchema_Call { + return &MaxComputeClient_ListSchema_Call{Call: _e.mock.On("ListSchema", ctx)} +} + +func (_c *MaxComputeClient_ListSchema_Call) Run(run func(ctx context.Context)) *MaxComputeClient_ListSchema_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MaxComputeClient_ListSchema_Call) Return(_a0 []*odps.Schema, _a1 error) *MaxComputeClient_ListSchema_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MaxComputeClient_ListSchema_Call) RunAndReturn(run func(context.Context) ([]*odps.Schema, error)) *MaxComputeClient_ListSchema_Call { + _c.Call.Return(run) + return _c +} + +// ListTable provides a mock function with given fields: ctx, schemaName +func (_m *MaxComputeClient) ListTable(ctx context.Context, schemaName string) ([]*odps.Table, error) { + ret := _m.Called(ctx, schemaName) + + if len(ret) == 0 { + panic("no return value specified for ListTable") + } + + var r0 []*odps.Table + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]*odps.Table, error)); ok { + return rf(ctx, schemaName) + } + if rf, ok := ret.Get(0).(func(context.Context, string) []*odps.Table); ok { + r0 = rf(ctx, schemaName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*odps.Table) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, schemaName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MaxComputeClient_ListTable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListTable' +type MaxComputeClient_ListTable_Call struct { + *mock.Call +} + +// ListTable is a helper method to define mock.On call +// - ctx context.Context +// - schemaName string +func (_e *MaxComputeClient_Expecter) ListTable(ctx interface{}, schemaName interface{}) *MaxComputeClient_ListTable_Call { + return &MaxComputeClient_ListTable_Call{Call: _e.mock.On("ListTable", ctx, schemaName)} +} + +func (_c *MaxComputeClient_ListTable_Call) Run(run func(ctx context.Context, schemaName string)) *MaxComputeClient_ListTable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MaxComputeClient_ListTable_Call) Return(_a0 []*odps.Table, _a1 error) *MaxComputeClient_ListTable_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MaxComputeClient_ListTable_Call) RunAndReturn(run func(context.Context, string) ([]*odps.Table, error)) *MaxComputeClient_ListTable_Call { + _c.Call.Return(run) + return _c +} + +// NewMaxComputeClient creates a new instance of MaxComputeClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMaxComputeClient(t interface { + mock.TestingT + Cleanup(func()) +}) *MaxComputeClient { + mock := &MaxComputeClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json b/plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json new file mode 100644 index 00000000..9d28f8d1 --- /dev/null +++ b/plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json @@ -0,0 +1,52 @@ +[ + { + "urn": "urn:maxcompute:test-project-id:table:test-project-id.my_schema.new_table", + "name": "new_table", + "service": "maxcompute", + "type": "MANAGED_TABLE", + "url": "", + "description": "", + "data": { + "@type": "type.googleapis.com/gotocompany.assets.v1beta2.Table", + "profile": null, + "columns": [ + { + "name": "user_id", + "description": "Unique identifier for users", + "data_type": "BIGINT", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": {} + }, + { + "name": "email", + "description": "User email address", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": {} + } + ], + "preview_fields": [], + "preview_rows": null, + "attributes": { + "project_name": "test-project-id", + "resource_url": "/projects/test-project-id/tables/new_table", + "schema": "my_schema", + "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table" + }, + "create_time": "2024-11-18T08:00:00Z", + "update_time": "2024-11-18T08:00:00Z" + }, + "owners": [], + "lineage": null, + "labels": {}, + "event": null, + "create_time": "2024-11-18T08:00:00Z", + "update_time": "2024-11-18T08:00:00Z" + } +] \ No newline at end of file diff --git a/plugins/extractors/maxcompute/testdata/expected-assets.json b/plugins/extractors/maxcompute/testdata/expected-assets.json new file mode 100644 index 00000000..ad5dbc8c --- /dev/null +++ b/plugins/extractors/maxcompute/testdata/expected-assets.json @@ -0,0 +1,123 @@ +[ + { + "urn": "urn:maxcompute:test-project-id:table:test-project-id.my_schema.dummy_table", + "name": "dummy_table", + "service": "maxcompute", + "type": "VIRTUAL_VIEW", + "url": "", + "description": "dummy table description", + "data": { + "@type": "type.googleapis.com/gotocompany.assets.v1beta2.Table", + "profile": null, + "columns": [ + { + "name": "id", + "description": "", + "data_type": "BIGINT", + "is_nullable": true, + "length": "0", + "profile": null, + "columns": [], + "attributes": {} + }, + { + "name": "name", + "description": "", + "data_type": "STRUCT", + "is_nullable": true, + "length": "0", + "profile": null, + "attributes": {}, + "columns": [ + { + "attributes": null, + "columns": [], + "data_type": "STRING", + "description": "", + "is_nullable": false, + "length": "0", + "name": "first_name", + "profile": null + }, + { + "attributes": null, + "columns": [], + "data_type": "STRING", + "description": "", + "is_nullable": false, + "length": "0", + "name": "last_name", + "profile": null + } + ] + } + ], + "preview_fields": [], + "preview_rows": null, + "attributes": { + "project_name": "test-project-id", + "resource_url": "/projects/test-project-id/tables/dummy_table", + "schema": "my_schema", + "sql": "SELECT id, name, user_info\nFROM test-project-id.default.my_dummy_table" + }, + "create_time": "2024-11-14T06:41:35Z", + "update_time": "2024-11-14T06:41:35Z" + }, + "owners": [], + "lineage": null, + "labels": {}, + "event": null, + "create_time": "2024-11-14T06:41:35Z", + "update_time": "2024-11-14T06:41:35Z" + }, + { + "urn": "urn:maxcompute:test-project-id:table:test-project-id.my_schema.new_table", + "name": "new_table", + "service": "maxcompute", + "type": "MANAGED_TABLE", + "url": "", + "description": "", + "data": { + "@type": "type.googleapis.com/gotocompany.assets.v1beta2.Table", + "profile": null, + "columns": [ + { + "name": "user_id", + "description": "Unique identifier for users", + "data_type": "BIGINT", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": {} + }, + { + "name": "email", + "description": "User email address", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": {} + } + ], + "preview_fields": [], + "preview_rows": null, + "attributes": { + "project_name": "test-project-id", + "resource_url": "/projects/test-project-id/tables/new_table", + "schema": "my_schema", + "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table" + }, + "create_time": "2024-11-18T08:00:00Z", + "update_time": "2024-11-18T08:00:00Z" + }, + "owners": [], + "lineage": null, + "labels": {}, + "event": null, + "create_time": "2024-11-18T08:00:00Z", + "update_time": "2024-11-18T08:00:00Z" + } +] \ No newline at end of file diff --git a/plugins/extractors/populate.go b/plugins/extractors/populate.go index 1597fbd0..351ff6cd 100644 --- a/plugins/extractors/populate.go +++ b/plugins/extractors/populate.go @@ -17,6 +17,7 @@ import ( _ "github.com/goto/meteor/plugins/extractors/http" _ "github.com/goto/meteor/plugins/extractors/kafka" _ "github.com/goto/meteor/plugins/extractors/mariadb" + _ "github.com/goto/meteor/plugins/extractors/maxcompute" _ "github.com/goto/meteor/plugins/extractors/merlin" _ "github.com/goto/meteor/plugins/extractors/metabase" _ "github.com/goto/meteor/plugins/extractors/mongodb" diff --git a/plugins/util.go b/plugins/util.go index adf180f0..fb518678 100644 --- a/plugins/util.go +++ b/plugins/util.go @@ -82,11 +82,6 @@ func MaxComputeTableFQNToURN(fqn string) (string, error) { return MaxComputeURN(projectName, schemaName, tableName), nil } -func MaxComputeURN(projectName, schemaName, tableName string) string { - fqn := fmt.Sprintf("%s.%s.%s", projectName, schemaName, tableName) - return models.NewURN("maxcompute", projectName, "table", fqn) -} - // BigQueryTableFQNToURN get URN from FQN (Fully Qualified Name) BigQuery func BigQueryTableFQNToURN(fqn string) (string, error) { projectID, datasetID, tableID, err := parseBQTableFQN(fqn) @@ -102,6 +97,12 @@ func BigQueryURN(projectID, datasetID, tableID string) string { return models.NewURN("bigquery", projectID, "table", fqn) } +//urn:maxcompute:project:schema:table +func MaxComputeURN(projectName, schemaName, tableName string) string { + fqn := fmt.Sprintf("%s.%s.%s", projectName, schemaName, tableName) + return models.NewURN("maxcompute", projectName, "table", fqn) +} + func KafkaURN(bootstrapServers, topic string) string { return models.NewURN("kafka", KafkaServersToScope(bootstrapServers), "topic", topic) }