[V0_11_To_Master] Fix td operator tests (treasure-data#1577)

#2a6516eb7fb3ceb746f5e362e49a32a56d7c4b41 #c7053debf5df9e2fe6605651d69a0ca5a98146b5 #3d4f50c22e172a173dda1ae68ccabb93ff799121 Co-authored-by: Mitsunori Komatsu <[email protected]>
yoyama · May 21, 2021 · 7badf25 · 7badf25
1 parent 4222d6c
commit 7badf25
Show file tree

Hide file tree

Showing 8 changed files with 300 additions and 95 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -83,13 +83,7 @@ workflows:
 
   test_and_docs_deployment:
     jobs:
-      - test: # master and v0_10 branches only
-          filters:
-            branches:
-              only:
-                - master
-                - v0_10
-                - v0_11
+      - test # will be executed for topic branch not only default branches so that we can safely merge it
       - docs_deployment: # master branch only
           filters:
             branches:

diff --git a/digdag-cli/src/main/java/io/digdag/cli/Run.java b/digdag-cli/src/main/java/io/digdag/cli/Run.java
@@ -222,21 +222,9 @@ public SystemExitException usage(String error)
 
     private static final List<Long> USE_ALL = null;
 
-    public void run(String workflowNameArg, String matchPattern)
-            throws Exception
+    protected DigdagEmbed.Bootstrap setupBootstrap(Properties systemProps)
     {
-        Properties systemProps = loadSystemProperties();
-
-        if (maxTaskThreads > 0) {
-            systemProps.setProperty("agent.max-task-threads", String.valueOf(maxTaskThreads));
-        }
-
-        if (taskLogPath != null) {
-            systemProps.setProperty("log-server.type", "local");
-            systemProps.setProperty("log-server.local.path", taskLogPath);
-        }
-
-        try (DigdagEmbed digdag = new DigdagEmbed.Bootstrap()
+        return new DigdagEmbed.Bootstrap()
                 .setEnvironment(env)
                 .setSystemConfig(PropertyUtils.toConfigElement(systemProps))
                 .setSystemPlugins(loadSystemPlugins(systemProps))
@@ -249,11 +237,25 @@ public void run(String workflowNameArg, String matchPattern)
                     binder.bind(Run.class).toProvider(() -> this);  // used by OperatorManagerWithSkip
                     binder.bind(AccessController.class).to(DefaultAccessController.class);
                 })
-                .overrideModulesWith((binder) -> {
-                    binder.bind(OperatorManager.class).to(OperatorManagerWithSkip.class).in(Scopes.SINGLETON);
-                })
-                .initializeWithoutShutdownHook()) {
+                .overrideModulesWith((binder) ->
+                        binder.bind(OperatorManager.class).to(OperatorManagerWithSkip.class).in(Scopes.SINGLETON));
+    }
+
+    public void run(String workflowNameArg, String matchPattern)
+            throws Exception
+    {
+        Properties systemProps = loadSystemProperties();
+
+        if (maxTaskThreads > 0) {
+            systemProps.setProperty("agent.max-task-threads", String.valueOf(maxTaskThreads));
+        }
+
+        if (taskLogPath != null) {
+            systemProps.setProperty("log-server.type", "local");
+            systemProps.setProperty("log-server.local.path", taskLogPath);
+        }
 
+        try (DigdagEmbed digdag = setupBootstrap(systemProps).initializeWithoutShutdownHook()) {
             run(systemProps, digdag.getInjector(), workflowNameArg, matchPattern);
         }
     }

diff --git a/digdag-tests/build.gradle b/digdag-tests/build.gradle
@@ -9,7 +9,7 @@ dependencies {
     testCompile 'com.squareup.okhttp3:okhttp:3.12.0'
     testCompile 'com.squareup.okhttp3:okhttp-tls:3.12.0'
     testCompile 'com.squareup.okhttp3:mockwebserver:3.12.0'
-    testCompile('xyz.rogfam:littleproxy:2.0.0-beta-4') {
+    testCompile('xyz.rogfam:littleproxy:2.0.3') {
         // littleproxy depends on guava:20 and it conflicts with digdag-client
         exclude group: 'com.google.guava', module: 'guava'
         // littleproxy depends on slf4j-api:1.7.24 and it conflicts with others

diff --git a/digdag-tests/src/test/java/acceptance/td/TdDdlIT.java b/digdag-tests/src/test/java/acceptance/td/TdDdlIT.java
@@ -165,7 +165,7 @@ public void stopServer()
     public void testRetries()
             throws Exception
     {
-        int failures = 7;
+        int failures = 3;
 
         Map<String, AtomicInteger> requests = new ConcurrentHashMap<>();
 
@@ -234,9 +234,14 @@ else if (fullHttpRequest.getUri().contains("v3/database/list")) {
                     }
                 }).start();
 
+        // In this test, the http proxy doesn't pass any request from `td` operator to TD API.
+        // So `td` operator needs to talk with the proxy over HTTP not HTTPS.
         Files.write(config, asList(
+                "config.td.min_retry_interval = 1s",
+                "config.td.max_retry_interval = 1s",
                 "params.td.use_ssl = false",
                 "params.td.proxy.enabled = true",
+                "params.td.proxy.use_ssl = true",
                 "params.td.proxy.host = " + proxyServer.getListenAddress().getHostString(),
                 "params.td.proxy.port = " + proxyServer.getListenAddress().getPort()
         ), APPEND);

diff --git a/digdag-tests/src/test/java/acceptance/td/TdForEachIT.java b/digdag-tests/src/test/java/acceptance/td/TdForEachIT.java
@@ -62,7 +62,7 @@ public void setUp()
                 .configuration(secretsServerConfiguration())
                 .configuration("config.td.wait.min_poll_interval = 5s")
                 .configuration(
-                        "params.td.use_ssl = false",
+                        "params.td.use_ssl = true",
                         "params.td.proxy.enabled = true",
                         "params.td.proxy.host = " + proxyServer.getListenAddress().getHostString(),
                         "params.td.proxy.port = " + proxyServer.getListenAddress().getPort()