diff --git a/go.mod b/go.mod index 582a0b07..08fac947 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,6 @@ require ( github.com/ClickHouse/clickhouse-go v1.4.5 github.com/IBM/sarama v1.43.2 github.com/MakeNowJust/heredoc v1.0.0 - github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.10 - github.com/alibabacloud-go/maxcompute-20220104 v1.4.1 github.com/aliyun/aliyun-odps-go-sdk v0.3.14 github.com/aws/aws-sdk-go v1.44.151 github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba @@ -85,14 +83,8 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/alecthomas/chroma v0.10.0 // indirect - github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5 // indirect github.com/alibabacloud-go/debug v1.0.1 // indirect - github.com/alibabacloud-go/endpoint-util v1.1.0 // indirect - github.com/alibabacloud-go/openapi-util v0.1.0 // indirect github.com/alibabacloud-go/tea v1.2.2 // indirect - github.com/alibabacloud-go/tea-utils v1.3.1 // indirect - github.com/alibabacloud-go/tea-utils/v2 v2.0.6 // indirect - github.com/alibabacloud-go/tea-xml v1.1.3 // indirect github.com/aliyun/credentials-go v1.3.10 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect @@ -115,7 +107,6 @@ require ( github.com/briandowns/spinner v1.20.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect - github.com/clbanning/mxj/v2 v2.5.5 // indirect github.com/cli/safeexec v1.0.1 // indirect github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/containerd/continuity v0.3.0 // indirect @@ -155,6 +146,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect @@ -225,7 +217,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.4.1 // indirect - github.com/tjfoc/gmsm v1.4.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect diff --git a/go.sum b/go.sum index e22ea27d..affa47aa 100644 --- a/go.sum +++ b/go.sum @@ -188,55 +188,13 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= -github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.6 h1:eIf+iGJxdU4U9ypaUfbtOWCsZSbTb8AUHvyPrxu6mAA= -github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.6/go.mod h1:4EUIoxs/do24zMOGGqYVWgw0s9NtiylnJglOeEB5UJo= -github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc= -github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5 h1:zE8vH9C7JiZLNJJQ5OwjU9mSi4T9ef9u3BURT6LCLC8= -github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5/go.mod h1:tWnyE9AjF8J8qqLk645oUmVUnFybApTQWklQmi5tY6g= -github.com/alibabacloud-go/darabonba-array v0.1.0 h1:vR8s7b1fWAQIjEjWnuF0JiKsCvclSRTfDzZHTYqfufY= -github.com/alibabacloud-go/darabonba-array v0.1.0/go.mod h1:BLKxr0brnggqOJPqT09DFJ8g3fsDshapUD3C3aOEFaI= -github.com/alibabacloud-go/darabonba-encode-util v0.0.2 h1:1uJGrbsGEVqWcWxrS9MyC2NG0Ax+GpOM5gtupki31XE= -github.com/alibabacloud-go/darabonba-encode-util v0.0.2/go.mod h1:JiW9higWHYXm7F4PKuMgEUETNZasrDM6vqVr/Can7H8= -github.com/alibabacloud-go/darabonba-map v0.0.2 h1:qvPnGB4+dJbJIxOOfawxzF3hzMnIpjmafa0qOTp6udc= -github.com/alibabacloud-go/darabonba-map v0.0.2/go.mod h1:28AJaX8FOE/ym8OUFWga+MtEzBunJwQGceGQlvaPGPc= -github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.8/go.mod h1:CzQnh+94WDnJOnKZH5YRyouL+OOcdBnXY5VWAf0McgI= -github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.10 h1:GEYkMApgpKEVDn6z12DcH1EGYpDYRB8JxsazM4Rywak= -github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.10/go.mod h1:26a14FGhZVELuz2cc2AolvW4RHmIO3/HRwsdHhaIPDE= -github.com/alibabacloud-go/darabonba-signature-util v0.0.7 h1:UzCnKvsjPFzApvODDNEYqBHMFt1w98wC7FOo0InLyxg= -github.com/alibabacloud-go/darabonba-signature-util v0.0.7/go.mod h1:oUzCYV2fcCH797xKdL6BDH8ADIHlzrtKVjeRtunBNTQ= -github.com/alibabacloud-go/darabonba-string v1.0.2 h1:E714wms5ibdzCqGeYJ9JCFywE5nDyvIXIIQbZVFkkqo= -github.com/alibabacloud-go/darabonba-string v1.0.2/go.mod h1:93cTfV3vuPhhEwGGpKKqhVW4jLe7tDpo3LUM0i0g6mA= -github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY= github.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= github.com/alibabacloud-go/debug v1.0.1 h1:MsW9SmUtbb1Fnt3ieC6NNZi6aEwrXfDksD4QA6GSbPg= github.com/alibabacloud-go/debug v1.0.1/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= -github.com/alibabacloud-go/endpoint-util v1.1.0 h1:r/4D3VSw888XGaeNpP994zDUaxdgTSHBbVfZlzf6b5Q= -github.com/alibabacloud-go/endpoint-util v1.1.0/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE= -github.com/alibabacloud-go/maxcompute-20220104 v1.4.1 h1:CwtCWRdr+OT581McaNKfgmSOSY8ShXBPrppMoUzbttw= -github.com/alibabacloud-go/maxcompute-20220104 v1.4.1/go.mod h1:qr4Zu1mwzYoiD6anCUz2wv1/VsNXdLkCtPQkRFaQKxE= -github.com/alibabacloud-go/openapi-util v0.1.0 h1:0z75cIULkDrdEhkLWgi9tnLe+KhAFE/r5Pb3312/eAY= -github.com/alibabacloud-go/openapi-util v0.1.0/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws= -github.com/alibabacloud-go/tea v1.1.0/go.mod h1:IkGyUSX4Ba1V+k4pCtJUc6jDpZLFph9QMy2VUPTwukg= -github.com/alibabacloud-go/tea v1.1.7/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= -github.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= -github.com/alibabacloud-go/tea v1.1.11/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= -github.com/alibabacloud-go/tea v1.1.17/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A= -github.com/alibabacloud-go/tea v1.1.20/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A= -github.com/alibabacloud-go/tea v1.2.1/go.mod h1:qbzof29bM/IFhLMtJPrgTGK3eauV5J2wSyEUo4OEmnA= github.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU= github.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk= -github.com/alibabacloud-go/tea-utils v1.3.1 h1:iWQeRzRheqCMuiF3+XkfybB3kTgUXkXX+JMrqfLeB2I= -github.com/alibabacloud-go/tea-utils v1.3.1/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE= -github.com/alibabacloud-go/tea-utils/v2 v2.0.5/go.mod h1:dL6vbUT35E4F4bFTHL845eUloqaerYBYPsdWR2/jhe4= -github.com/alibabacloud-go/tea-utils/v2 v2.0.6 h1:ZkmUlhlQbaDC+Eba/GARMPy6hKdCLiSke5RsN5LcyQ0= -github.com/alibabacloud-go/tea-utils/v2 v2.0.6/go.mod h1:qxn986l+q33J5VkialKMqT/TTs3E+U9MJpd001iWQ9I= -github.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0= -github.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= github.com/aliyun/aliyun-odps-go-sdk v0.3.14 h1:H+p9+2n4kSBOiJvndKC7M+0UxK0mW4azQHQzHy2hI2A= github.com/aliyun/aliyun-odps-go-sdk v0.3.14/go.mod h1:t/tgF/iN5aAs/gLL7sEI8/qdax4NuFCKEjO3OJbHZqI= -github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= -github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= -github.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM= github.com/aliyun/credentials-go v1.3.10 h1:45Xxrae/evfzQL9V10zL3xX31eqgLWEaIdCoPipOEQA= github.com/aliyun/credentials-go v1.3.10/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= @@ -390,8 +348,6 @@ github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/clbanning/mxj/v2 v2.5.5 h1:oT81vUeEiQQ/DcHbzSytRngP6Ky9O+L+0Bw0zSJag9E= -github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q= github.com/cli/safeexec v1.0.1 h1:e/C79PbXF4yYTN/wauC4tviMxEV13BwljGj0N9j+N00= github.com/cli/safeexec v1.0.1/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q= @@ -1507,7 +1463,6 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/snowflakedb/gosnowflake v1.6.3/go.mod h1:6hLajn6yxuJ4xUHZegMekpq9rnQbGJ7TMwXjgTmA6lg= @@ -1582,9 +1537,6 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= -github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= -github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -1637,7 +1589,6 @@ github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCO github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.3/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -1778,16 +1729,13 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -1800,11 +1748,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1912,7 +1857,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1954,11 +1898,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2064,7 +2005,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2159,12 +2099,9 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2177,11 +2114,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2199,7 +2133,6 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -2268,7 +2201,6 @@ golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -2548,7 +2480,6 @@ gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:a gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/plugins/extractors/maxcompute/client.go b/plugins/extractors/maxcompute/client.go new file mode 100644 index 00000000..9cc238d2 --- /dev/null +++ b/plugins/extractors/maxcompute/client.go @@ -0,0 +1,64 @@ +package maxcompute + +import ( + "context" + + "github.com/aliyun/aliyun-odps-go-sdk/odps" + "github.com/aliyun/aliyun-odps-go-sdk/odps/account" +) + +type MaxComputeClient struct { + client *odps.Odps + project *odps.Project +} + +func NewMaxComputeClient(config Config) *MaxComputeClient { + aliAccount := account.NewAliyunAccount(config.AccessKey.ID, config.AccessKey.Secret) + client := odps.NewOdps(aliAccount, config.EndpointProject) + client.SetDefaultProjectName(config.ProjectName) + + project := client.Project(config.ProjectName) + + return &MaxComputeClient{ + client: client, + project: project, + } +} + +func (c *MaxComputeClient) ListSchema(ctx context.Context) (schemas []*odps.Schema, err error) { + err = c.project.Schemas().List(func(schema *odps.Schema, err2 error) { + if err2 != nil { + err = err2 + return + } + schemas = append(schemas, schema) + }) + + return +} + +func (c *MaxComputeClient) ListTable(ctx context.Context, schemaName string) (tables []*odps.Table, err error) { + t := odps.NewTables(c.client, c.project.Name(), schemaName) + t.List( + func(table *odps.Table, err2 error) { + if err2 != nil { + err = err2 + return + } + tables = append(tables, table) + }, + ) + return +} + +func (c *MaxComputeClient) GetTable(ctx context.Context, table *odps.Table) (*odps.Table, error) { + err := table.Load() + if err != nil { + isView := table.Schema().IsVirtualView || table.Schema().IsMaterializedView + isLoaded := table.IsLoaded() + if !isView || (isView && !isLoaded) { + return nil, err + } + } + return table, nil +} diff --git a/plugins/extractors/maxcompute/maxcompute.go b/plugins/extractors/maxcompute/maxcompute.go index 74fbd862..728e49c3 100644 --- a/plugins/extractors/maxcompute/maxcompute.go +++ b/plugins/extractors/maxcompute/maxcompute.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/aliyun/aliyun-odps-go-sdk/odps" - "github.com/aliyun/aliyun-odps-go-sdk/odps/account" "github.com/aliyun/aliyun-odps-go-sdk/odps/datatype" "github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema" "github.com/goto/meteor/models" @@ -40,10 +39,13 @@ type Extractor struct { logger log.Logger config Config - client *odps.Odps - eg *errgroup.Group + client Client + newClient NewClientFunc + eg *errgroup.Group } +type NewClientFunc func(ctx context.Context, logger log.Logger, config Config) (Client, error) + //go:embed README.md var summary string @@ -70,9 +72,17 @@ var info = plugins.Info{ Summary: summary, } -func New(logger log.Logger) *Extractor { +//go:generate mockery --name=Client -r --case underscore --with-expecter --structname MaxComputeClient --filename maxcompute_client_mock.go --output=./mocks +type Client interface { + ListSchema(ctx context.Context) ([]*odps.Schema, error) + ListTable(ctx context.Context, schemaName string) ([]*odps.Table, error) + GetTable(ctx context.Context, table *odps.Table) (*odps.Table, error) +} + +func New(logger log.Logger, clientFunc NewClientFunc) *Extractor { e := &Extractor{ - logger: logger, + logger: logger, + newClient: clientFunc, } e.BaseExtractor = plugins.NewBaseExtractor(info, &e.config) e.ScopeNotRequired = true @@ -85,9 +95,24 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) error { return err } - aliAccount := account.NewAliyunAccount(e.config.AccessKey.ID, e.config.AccessKey.Secret) - e.client = odps.NewOdps(aliAccount, e.config.EndpointProject) - e.client.SetDefaultProjectName(e.config.ProjectName) + if e.config.ProjectName == "" { + return fmt.Errorf("project_name is required") + } + if e.config.AccessKey.ID == "" || e.config.AccessKey.Secret == "" { + return fmt.Errorf("access_key is required") + } + if e.config.EndpointProject == "" { + return fmt.Errorf("endpoint_project is required") + } + if e.config.Concurrency == 0 { + e.config.Concurrency = 1 + } + + var err error + e.client, err = e.newClient(ctx, e.logger, e.config) + if err != nil { + return err + } e.eg = &errgroup.Group{} e.eg.SetLimit(e.config.Concurrency) @@ -95,59 +120,55 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) error { return nil } -func (e *Extractor) Extract(_ context.Context, emit plugins.Emit) error { - schemas := e.client.Project(e.config.ProjectName).Schemas() +func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) error { - err := schemas.List(func(schema *odps.Schema, err error) { - if err != nil { - e.logger.Error("failed to list schemas", "error", err) - return - } + schemas, err := e.client.ListSchema(ctx) + + if err != nil { + return err + } + + for _, schema := range schemas { if e.config.SchemaName != "" && schema.Name() != e.config.SchemaName { - return + continue } if contains(e.config.Exclude.Schemas, schema.Name()) { - return + continue } - tables := odps.NewTables(e.client, e.config.ProjectName, schema.Name()) + err := e.fetchTablesFromSchema(ctx, schema, emit) + if err != nil { + return err + } + } + + return e.eg.Wait() +} - e.fetchTablesFromSchema(tables, emit) - }) +func (e *Extractor) fetchTablesFromSchema(ctx context.Context, schema *odps.Schema, emit plugins.Emit) error { + + tables, err := e.client.ListTable(ctx, schema.Name()) if err != nil { return err } - return e.eg.Wait() -} -func (e *Extractor) fetchTablesFromSchema(tables *odps.Tables, emit plugins.Emit) { - tables.List( - func(t *odps.Table, err error) { - if err != nil { - e.logger.Error("table list", err) - return - } - - if contains(e.config.Exclude.Tables, fmt.Sprintf("%s.%s", t.SchemaName(), t.Name())) { - return - } - - e.eg.Go(func() error { - return e.processTable(t, emit) - }) - }, - ) + for _, table := range tables { + if contains(e.config.Exclude.Tables, fmt.Sprintf("%s.%s", table.SchemaName(), table.Name())) { + continue + } + + e.eg.Go(func() error { + return e.processTable(ctx, table, emit) + }) + } + + return nil } -func (e *Extractor) processTable(table *odps.Table, emit plugins.Emit) error { - err := table.Load() +func (e *Extractor) processTable(ctx context.Context, table *odps.Table, emit plugins.Emit) error { + table, err := e.client.GetTable(ctx, table) if err != nil { - isView := table.Schema().IsVirtualView || table.Schema().IsMaterializedView - isLoaded := table.IsLoaded() - if !isView || (isView && !isLoaded) { - e.logger.Error("failed to get table info", "schema", table.SchemaName(), "table", table.Name(), "error", err) - return err - } + return err } asset, err := e.buildAsset(table) @@ -256,6 +277,10 @@ func buildTableAttributesData(tableInfo *odps.Table) map[string]interface{} { attributesData["resource_url"] = tableInfo.ResourceUrl() } + // if tableInfo.PartitionColumns() != nil { + // attributesData["partition_field"] = tableInfo.PartitionColumns() + // } + return attributesData } @@ -294,8 +319,12 @@ func contains(slice []string, item string) bool { func init() { if err := registry.Extractors.Register("maxcompute", func() plugins.Extractor { - return New(plugins.GetLog()) + return New(plugins.GetLog(), CreateClient) }); err != nil { panic(err) } } + +func CreateClient(ctx context.Context, logger log.Logger, config Config) (Client, error) { + return NewMaxComputeClient(config), nil +} diff --git a/plugins/extractors/maxcompute/maxcompute_test.go b/plugins/extractors/maxcompute/maxcompute_test.go new file mode 100644 index 00000000..c8e36244 --- /dev/null +++ b/plugins/extractors/maxcompute/maxcompute_test.go @@ -0,0 +1,238 @@ +//go:build plugins +// +build plugins + +package maxcompute_test + +import ( + "context" + "fmt" + "testing" + "time" + + // "github.com/aliyun/aliyun-odps-go-sdk/odps" + "github.com/aliyun/aliyun-odps-go-sdk/odps" + v1beta2 "github.com/goto/meteor/models/gotocompany/assets/v1beta2" + "github.com/goto/meteor/plugins" + "github.com/goto/meteor/plugins/extractors/maxcompute" + "github.com/goto/meteor/plugins/extractors/maxcompute/mocks" + mocks2 "github.com/goto/meteor/test/mocks" + "github.com/goto/meteor/test/utils" + "github.com/goto/salt/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + projectID = "test-project-id" +) + +// func mockClient(ctx context.Context, logger slog.Logger, config *maxcompute.Config) (maxcompute.Client, error) { +// client := mocks2.NewMaxComputeClient() +// return client, nil +// } + +func TestInit(t *testing.T) { + t.Run("should return error if config is invalid", func(t *testing.T) { + extr := maxcompute.New(utils.Logger, maxcompute.CreateClient) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := extr.Init(ctx, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": "", + }, + }) + + assert.ErrorContains(t, err, "project_name is required") + }) + + t.Run("should return no error", func(t *testing.T) { + extr := maxcompute.New(utils.Logger, maxcompute.CreateClient) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := extr.Init(ctx, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": projectID, + "access_key": map[string]interface{}{ + "id": "access_key_id", + "secret": "access_key_secret", + }, + "endpoint_project": "https://example.com/some-api", + }, + }) + + assert.NoError(t, err) + }) +} + +func createClient(client *mocks.MaxComputeClient) func(ctx context.Context, logger log.Logger, config maxcompute.Config) (maxcompute.Client, error) { + return func(ctx context.Context, logger log.Logger, config maxcompute.Config) (maxcompute.Client, error) { + return client, nil + } + +} + +func TestExtract(t *testing.T) { + + schema1 := []*odps.Schema{ + odps.NewSchema(nil, projectID, "my_schema"), + } + + table1 := []*odps.Table{ + odps.NewTable(nil, projectID, "my_schema", "my_dummy_table"), + } + + runTest := func(t *testing.T, cfg plugins.Config, mockSetup func(mockClient *mocks.MaxComputeClient)) ([]*v1beta2.Asset, error) { + mockClient := mocks.NewMaxComputeClient(t) + if mockSetup != nil { + mockSetup(mockClient) + } + extr := maxcompute.New(utils.Logger, createClient(mockClient)) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + err := extr.Init(ctx, cfg) + if err != nil { + return nil, err + } + + emitter := mocks2.NewEmitter() + err = extr.Extract(ctx, emitter.Push) + + actual := getAllData(emitter, t) + return actual, err + } + + t.Run("should return no error", func(t *testing.T) { + actual, err := runTest(t, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": projectID, + "access_key": map[string]interface{}{ + "id": "access_key_id", + "secret": "access_key_secret", + }, + "endpoint_project": "https://example.com/some-api", + }, + }, func(mockClient *mocks.MaxComputeClient) { + mockClient.EXPECT().ListSchema(mock.Anything).Return(schema1, nil) + mockClient.EXPECT().ListTable(mock.Anything, "my_schema").Return(table1, nil) + mockClient.EXPECT().GetTable(mock.Anything, table1[0]).Return(table1[0], nil) + }) + + assert.Nil(t, err) + assert.NotNil(t, actual) + utils.AssertProtosWithJSONFile(t, "testdata/expected-assets.json", actual) + }) + + // t.Run("should exclude schemas based on exclusion rules", func(t *testing.T) { + // actual := runTest(t, plugins.Config{ + // URNScope: "test-maxcompute", + // RawConfig: map[string]interface{}{ + // "project_name": projectID, + // "access_key": map[string]interface{}{ + // "id": "access_key_id", + // "secret": "access_key_secret", + // }, + // "endpoint_project": "https://example.com/some-api", + // "exclude_schemas": []string{"schema2"}, + // }, + // }, func(mockClient *mocks.MaxComputeClient) { + // mockClient.EXPECT().ListSchema(mock.Anything).Return([]*odps.Schema{ + // { + // Name: "schema1", + // Tables: []*odps.Table{ + // {Name: "table1", Comment: "Test table 1"}, + // {Name: "table2", Comment: "Test table 2"}, + // }, + // }, + // { + // Name: "schema2", + // Tables: []*odps.Table{ + // {Name: "table3", Comment: "Test table 3"}, + // }, + // }, + // }, nil) + // }) + + // utils.AssertProtosWithJSONFile(t, "testdata/expected-assets-excluded-schema.json", actual) + // }) + + // t.Run("should exclude tables based on exclusion rules", func(t *testing.T) { + // actual := runTest(t, plugins.Config{ + // URNScope: "test-maxcompute", + // RawConfig: map[string]interface{}{ + // "project_name": projectID, + // "access_key": map[string]interface{}{ + // "id": "access_key_id", + // "secret": "access_key_secret", + // }, + // "endpoint_project": "https://example.com/some-api", + // "exclude_tables": []string{"schema1.table2"}, + // }, + // }, func(mockClient *mocks.MaxComputeClient) { + // mockClient.EXPECT().ListSchema(mock.Anything).Return([]*odps.Schema{ + // { + // Name: "schema1", + // Tables: []*odps.Table{ + // {Name: "table1", Comment: "Test table 1"}, + // {Name: "table2", Comment: "Test table 2"}, + // }, + // }, + // }, nil) + // }) + + // utils.AssertProtosWithJSONFile(t, "testdata/expected-assets-excluded-table.json", actual) + // }) + + t.Run("should return error if ListSchema fails", func(t *testing.T) { + actual, err := runTest(t, plugins.Config{ + URNScope: "test-maxcompute", + RawConfig: map[string]interface{}{ + "project_name": projectID, + "access_key": map[string]interface{}{ + "id": "access_key_id", + "secret": "access_key_secret", + }, + "endpoint_project": "https://example.com/some-api", + }, + }, func(mockClient *mocks.MaxComputeClient) { + mockClient.EXPECT().ListSchema(mock.Anything).Return(nil, fmt.Errorf("ListSchema fails")) + }) + assert.ErrorContains(t, err, "ListSchema fails") + assert.Nil(t, actual) + }) +} + +func getAllData(emitter *mocks2.Emitter, t *testing.T) []*v1beta2.Asset { + actual := emitter.GetAllData() + + // the emulator appending 1 random dataset + // we can't assert it, so we remove it from the list + if len(actual) > 0 { + actual = actual[:len(actual)-1] + } + + // the emulator returning dynamic timestamps + // replace them with static ones + for _, asset := range actual { + replaceWithStaticTimestamp(t, asset) + } + return actual +} + +func replaceWithStaticTimestamp(t *testing.T, asset *v1beta2.Asset) { + b := new(v1beta2.Table) + err := asset.Data.UnmarshalTo(b) + assert.NoError(t, err) + + time, err := time.Parse(time.RFC3339, "2023-06-13T03:46:12.372974Z") + assert.NoError(t, err) + b.CreateTime = timestamppb.New(time) + b.UpdateTime = timestamppb.New(time) + + asset.Data, err = anypb.New(b) + assert.NoError(t, err) +} diff --git a/plugins/extractors/maxcompute/mocks/maxcompute_client_mock.go b/plugins/extractors/maxcompute/mocks/maxcompute_client_mock.go new file mode 100644 index 00000000..c13f44cd --- /dev/null +++ b/plugins/extractors/maxcompute/mocks/maxcompute_client_mock.go @@ -0,0 +1,214 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + odps "github.com/aliyun/aliyun-odps-go-sdk/odps" +) + +// MaxComputeClient is an autogenerated mock type for the Client type +type MaxComputeClient struct { + mock.Mock +} + +type MaxComputeClient_Expecter struct { + mock *mock.Mock +} + +func (_m *MaxComputeClient) EXPECT() *MaxComputeClient_Expecter { + return &MaxComputeClient_Expecter{mock: &_m.Mock} +} + +// GetTable provides a mock function with given fields: ctx, table +func (_m *MaxComputeClient) GetTable(ctx context.Context, table *odps.Table) (*odps.Table, error) { + ret := _m.Called(ctx, table) + + if len(ret) == 0 { + panic("no return value specified for GetTable") + } + + var r0 *odps.Table + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *odps.Table) (*odps.Table, error)); ok { + return rf(ctx, table) + } + if rf, ok := ret.Get(0).(func(context.Context, *odps.Table) *odps.Table); ok { + r0 = rf(ctx, table) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*odps.Table) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *odps.Table) error); ok { + r1 = rf(ctx, table) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MaxComputeClient_GetTable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTable' +type MaxComputeClient_GetTable_Call struct { + *mock.Call +} + +// GetTable is a helper method to define mock.On call +// - ctx context.Context +// - table *odps.Table +func (_e *MaxComputeClient_Expecter) GetTable(ctx interface{}, table interface{}) *MaxComputeClient_GetTable_Call { + return &MaxComputeClient_GetTable_Call{Call: _e.mock.On("GetTable", ctx, table)} +} + +func (_c *MaxComputeClient_GetTable_Call) Run(run func(ctx context.Context, table *odps.Table)) *MaxComputeClient_GetTable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*odps.Table)) + }) + return _c +} + +func (_c *MaxComputeClient_GetTable_Call) Return(_a0 *odps.Table, _a1 error) *MaxComputeClient_GetTable_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MaxComputeClient_GetTable_Call) RunAndReturn(run func(context.Context, *odps.Table) (*odps.Table, error)) *MaxComputeClient_GetTable_Call { + _c.Call.Return(run) + return _c +} + +// ListSchema provides a mock function with given fields: ctx +func (_m *MaxComputeClient) ListSchema(ctx context.Context) ([]*odps.Schema, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for ListSchema") + } + + var r0 []*odps.Schema + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*odps.Schema, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []*odps.Schema); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*odps.Schema) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MaxComputeClient_ListSchema_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListSchema' +type MaxComputeClient_ListSchema_Call struct { + *mock.Call +} + +// ListSchema is a helper method to define mock.On call +// - ctx context.Context +func (_e *MaxComputeClient_Expecter) ListSchema(ctx interface{}) *MaxComputeClient_ListSchema_Call { + return &MaxComputeClient_ListSchema_Call{Call: _e.mock.On("ListSchema", ctx)} +} + +func (_c *MaxComputeClient_ListSchema_Call) Run(run func(ctx context.Context)) *MaxComputeClient_ListSchema_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MaxComputeClient_ListSchema_Call) Return(_a0 []*odps.Schema, _a1 error) *MaxComputeClient_ListSchema_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MaxComputeClient_ListSchema_Call) RunAndReturn(run func(context.Context) ([]*odps.Schema, error)) *MaxComputeClient_ListSchema_Call { + _c.Call.Return(run) + return _c +} + +// ListTable provides a mock function with given fields: ctx, schemaName +func (_m *MaxComputeClient) ListTable(ctx context.Context, schemaName string) ([]*odps.Table, error) { + ret := _m.Called(ctx, schemaName) + + if len(ret) == 0 { + panic("no return value specified for ListTable") + } + + var r0 []*odps.Table + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]*odps.Table, error)); ok { + return rf(ctx, schemaName) + } + if rf, ok := ret.Get(0).(func(context.Context, string) []*odps.Table); ok { + r0 = rf(ctx, schemaName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*odps.Table) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, schemaName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MaxComputeClient_ListTable_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListTable' +type MaxComputeClient_ListTable_Call struct { + *mock.Call +} + +// ListTable is a helper method to define mock.On call +// - ctx context.Context +// - schemaName string +func (_e *MaxComputeClient_Expecter) ListTable(ctx interface{}, schemaName interface{}) *MaxComputeClient_ListTable_Call { + return &MaxComputeClient_ListTable_Call{Call: _e.mock.On("ListTable", ctx, schemaName)} +} + +func (_c *MaxComputeClient_ListTable_Call) Run(run func(ctx context.Context, schemaName string)) *MaxComputeClient_ListTable_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MaxComputeClient_ListTable_Call) Return(_a0 []*odps.Table, _a1 error) *MaxComputeClient_ListTable_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MaxComputeClient_ListTable_Call) RunAndReturn(run func(context.Context, string) ([]*odps.Table, error)) *MaxComputeClient_ListTable_Call { + _c.Call.Return(run) + return _c +} + +// NewMaxComputeClient creates a new instance of MaxComputeClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMaxComputeClient(t interface { + mock.TestingT + Cleanup(func()) +}) *MaxComputeClient { + mock := &MaxComputeClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/plugins/extractors/maxcompute/testdata/expected-assets.json b/plugins/extractors/maxcompute/testdata/expected-assets.json new file mode 100644 index 00000000..fd89ec2a --- /dev/null +++ b/plugins/extractors/maxcompute/testdata/expected-assets.json @@ -0,0 +1,186 @@ +[ + { + "urn": "urn:maxcompute:dummy_project:table:dummy_project.my_schema.dummy_table_view", + "name": "test-project-id", + "service": "maxcompute", + "type": "VIRTUAL_VIEW", + "url": "", + "description": "", + "data": { + "@type": "type.googleapis.com/gotocompany.assets.v1beta2.Table", + "profile": null, + "columns": [ + { + "name": "id", + "description": "", + "data_type": "BIGINT", + "is_nullable": true, + "length": "0", + "profile": null, + "columns": [], + "attributes": {} + }, + { + "name": "name", + "description": "", + "data_type": "STRING", + "is_nullable": true, + "length": "0", + "profile": null, + "columns": [], + "attributes": {} + }, + { + "name": "user_info", + "description": "", + "data_type": "STRUCT", + "is_nullable": true, + "length": "0", + "profile": null, + "columns": [ + { + "name": "user_id", + "description": "", + "data_type": "BIGINT", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + }, + { + "name": "personal", + "description": "", + "data_type": "STRUCT", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [ + { + "name": "first_name", + "description": "", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + }, + { + "name": "last_name", + "description": "", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + }, + { + "name": "details", + "description": "", + "data_type": "STRUCT", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [ + { + "name": "birth_date", + "description": "", + "data_type": "DATE", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + }, + { + "name": "gender", + "description": "", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + }, + { + "name": "nationality", + "description": "", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + } + ], + "attributes": null + } + ], + "attributes": null + }, + { + "name": "contact", + "description": "", + "data_type": "STRUCT", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [ + { + "name": "email", + "description": "", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + }, + { + "name": "secondary_email", + "description": "", + "data_type": "STRING", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + }, + { + "name": "phones", + "description": "", + "data_type": "ARRAY", + "is_nullable": false, + "length": "0", + "profile": null, + "columns": [], + "attributes": null + } + ], + "attributes": null + } + ], + "attributes": {} + } + ], + "preview_fields": [], + "preview_rows": null, + "attributes": { + "project_name": "dummy_project", + "resource_url": "/projects/dummy_project/tables/dummy_table_view", + "schema": "my_schema", + "sql": "SELECT id, name, user_info\nFROM dummy_project.default.my_dummy_table" + }, + "create_time": "2024-11-14T06:41:35Z", + "update_time": "2024-11-14T06:41:35Z" + }, + "owners": [], + "lineage": null, + "labels": {}, + "event": null, + "create_time": "2024-11-14T06:41:35Z", + "update_time": "2024-11-14T06:41:35Z" + } +] \ No newline at end of file