From e25a32d045b8689174efb8f3046ba19a5c56dfb6 Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Thu, 23 Jan 2025 20:56:08 -0800 Subject: [PATCH 1/3] ATLAS-4937: update checkstyle to improve code readability (#269) --- build-tools/src/main/resources/checkstyle.xml | 240 --------- dev-support/AtlasCodeScheme-IntelliJ.xml | 501 ++++++++++++++++++ .../checkstyle-suppressions.xml | 6 +- dev-support/checkstyle.xml | 272 ++++++++++ pom.xml | 22 +- 5 files changed, 790 insertions(+), 251 deletions(-) delete mode 100755 build-tools/src/main/resources/checkstyle.xml create mode 100644 dev-support/AtlasCodeScheme-IntelliJ.xml rename {build-tools/src/main/resources => dev-support}/checkstyle-suppressions.xml (79%) create mode 100755 dev-support/checkstyle.xml diff --git a/build-tools/src/main/resources/checkstyle.xml b/build-tools/src/main/resources/checkstyle.xml deleted file mode 100755 index 54423176f5..0000000000 --- a/build-tools/src/main/resources/checkstyle.xml +++ /dev/null @@ -1,240 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dev-support/AtlasCodeScheme-IntelliJ.xml b/dev-support/AtlasCodeScheme-IntelliJ.xml new file mode 100644 index 0000000000..f3edfb2a0f --- /dev/null +++ b/dev-support/AtlasCodeScheme-IntelliJ.xml @@ -0,0 +1,501 @@ + + + + + diff --git a/build-tools/src/main/resources/checkstyle-suppressions.xml b/dev-support/checkstyle-suppressions.xml similarity index 79% rename from build-tools/src/main/resources/checkstyle-suppressions.xml rename to dev-support/checkstyle-suppressions.xml index 759a52ec75..d56f4a1436 100644 --- a/build-tools/src/main/resources/checkstyle-suppressions.xml +++ b/dev-support/checkstyle-suppressions.xml @@ -24,6 +24,8 @@ - - + + + + diff --git a/dev-support/checkstyle.xml b/dev-support/checkstyle.xml new file mode 100755 index 0000000000..349971428c --- /dev/null +++ b/dev-support/checkstyle.xml @@ -0,0 +1,272 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 0f5e6acf70..ef2d323c9d 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,9 @@ 1.7.5 1.16.0 false + 3.1.0 + true + 8.29 3.1.4 1.4 1.14 @@ -1162,12 +1165,6 @@ 0.13 - - org.apache.maven.plugins - maven-checkstyle-plugin - 2.9.1 - - org.apache.maven.plugins maven-site-plugin @@ -1505,7 +1502,13 @@ org.apache.maven.plugins maven-checkstyle-plugin + ${checkstyle.plugin.version} + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + org.apache.atlas atlas-buildtools @@ -1520,10 +1523,11 @@ verify - ${skipCheck} - false + true true - checkstyle.suppressions.file + ./dev-support/checkstyle.xml + ./dev-support/checkstyle-suppressions.xml + ${checkstyle.skip} ${checkstyle.failOnViolation} From 1d322d34bb4bf6307fe5f90603ce07d0035001aa Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Thu, 23 Jan 2025 23:42:46 -0800 Subject: [PATCH 2/3] .asf.yaml updated with collaborators --- .asf.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index deea72d81f..634bf0f80e 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -40,3 +40,11 @@ github: jira_options: worklog link label protected_branches: master: {} + collaborators: + - ashutoshm + - DishaTalreja3 + - kevalbhatt + - nixonrodrigues + - pinal-shah + - rkundam + - sarathsubramanian From 155c854c6cb113e190740e17665179526a079f1f Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Thu, 23 Jan 2025 23:44:22 -0800 Subject: [PATCH 3/3] ATLAS-4957: checkstyle compliance updates - atlas-common module (#272) --- common/pom.xml | 5 + .../java/org/apache/atlas/AtlasConstants.java | 16 +- .../ConditionalOnAtlasProperty.java | 3 +- .../org/apache/atlas/annotation/Timed.java | 11 +- .../groovy/AbstractFunctionExpression.java | 22 +- .../groovy/AbstractGroovyExpression.java | 15 +- .../atlas/groovy/ArithmeticExpression.java | 46 +-- .../apache/atlas/groovy/BinaryExpression.java | 21 +- .../apache/atlas/groovy/CastExpression.java | 20 +- .../atlas/groovy/ClosureExpression.java | 85 +++--- .../atlas/groovy/ComparisonExpression.java | 51 ++-- .../groovy/ComparisonOperatorExpression.java | 8 +- .../apache/atlas/groovy/FieldExpression.java | 13 +- .../atlas/groovy/FunctionCallExpression.java | 51 ++-- .../apache/atlas/groovy/GroovyExpression.java | 8 +- .../atlas/groovy/GroovyGenerationContext.java | 20 +- .../atlas/groovy/IdentifierExpression.java | 27 +- .../atlas/groovy/LabeledExpression.java | 15 +- .../apache/atlas/groovy/ListExpression.java | 18 +- .../atlas/groovy/LiteralExpression.java | 99 +++---- .../atlas/groovy/LogicalExpression.java | 38 +-- .../apache/atlas/groovy/RangeExpression.java | 24 +- .../atlas/groovy/StatementListExpression.java | 31 +- .../groovy/TernaryOperatorExpression.java | 37 ++- .../atlas/groovy/TraversalStepType.java | 6 +- .../atlas/groovy/TypeCoersionExpression.java | 18 +- .../groovy/VariableAssignmentExpression.java | 25 +- .../org/apache/atlas/ha/HAConfiguration.java | 154 +++++----- .../org/apache/atlas/pc/StatusReporter.java | 42 ++- .../org/apache/atlas/pc/WorkItemBuilder.java | 6 +- .../org/apache/atlas/pc/WorkItemConsumer.java | 20 +- .../org/apache/atlas/pc/WorkItemManager.java | 67 +++-- .../apache/atlas/repository/Constants.java | 105 +++---- .../org/apache/atlas/service/Service.java | 7 +- .../org/apache/atlas/service/Services.java | 4 +- .../atlas/utils/AtlasPathExtractorUtil.java | 181 +++++------- .../apache/atlas/utils/AtlasPerfMetrics.java | 44 ++- .../apache/atlas/utils/AtlasPerfTracer.java | 28 +- .../atlas/utils/HdfsNameServiceResolver.java | 77 ++--- .../org/apache/atlas/utils/KafkaUtils.java | 277 ++++++++---------- .../java/org/apache/atlas/utils/LruCache.java | 44 +-- .../atlas/utils/OnAtlasPropertyCondition.java | 21 +- .../org/apache/atlas/utils/ParamChecker.java | 19 +- .../atlas/utils/PathExtractorContext.java | 8 +- .../apache/atlas/utils/PropertiesUtil.java | 100 ++++--- .../org/apache/atlas/utils/SHA256Utils.java | 48 +-- .../apache/atlas/ha/HAConfigurationTest.java | 24 +- .../utils/AtlasPathExtractorUtilTest.java | 240 +++++++-------- .../utils/HdfsNameServiceResolverTest.java | 4 +- .../apache/atlas/utils/KafkaUtilsTest.java | 209 ++++++------- .../org/apache/atlas/utils/LruCacheTest.java | 71 ++--- .../apache/atlas/pc/StatusReporterTest.java | 36 ++- .../apache/atlas/pc/WorkItemManagerTest.java | 14 +- .../pc/WorkItemManagerWithResultsTest.java | 41 ++- 54 files changed, 1291 insertions(+), 1333 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 0e80cfe002..d722f55911 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -31,6 +31,11 @@ Apache Atlas Common Apache Atlas Common + + true + false + + diff --git a/common/src/main/java/org/apache/atlas/AtlasConstants.java b/common/src/main/java/org/apache/atlas/AtlasConstants.java index ce0dbd0b4e..cc0d507633 100644 --- a/common/src/main/java/org/apache/atlas/AtlasConstants.java +++ b/common/src/main/java/org/apache/atlas/AtlasConstants.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,9 +22,6 @@ * Constants used in Atlas configuration. */ public final class AtlasConstants { - private AtlasConstants() { - } - public static final String CLUSTER_NAME_KEY = "atlas.cluster.name"; public static final String METADATA_NAMESPACE_KEY = "atlas.metadata.namespace"; public static final String DEFAULT_CLUSTER_NAME = "primary"; @@ -32,10 +29,13 @@ private AtlasConstants() { public static final String ATLAS_REST_ADDRESS_KEY = "atlas.rest.address"; public static final String ATLAS_MIGRATION_MODE_FILENAME = "atlas.migration.data.filename"; public static final String ATLAS_SERVICES_ENABLED = "atlas.services.enabled"; - public static final String CLUSTER_NAME_ATTRIBUTE = "clusterName"; public static final String DEFAULT_APP_PORT_STR = "21000"; public static final String DEFAULT_ATLAS_REST_ADDRESS = "http://localhost:21000"; public static final String DEFAULT_TYPE_VERSION = "1.0"; public static final int ATLAS_SHUTDOWN_HOOK_PRIORITY = 30; -} \ No newline at end of file + + private AtlasConstants() { + // to block instantiation + } +} diff --git a/common/src/main/java/org/apache/atlas/annotation/ConditionalOnAtlasProperty.java b/common/src/main/java/org/apache/atlas/annotation/ConditionalOnAtlasProperty.java index 427e3a8f84..db038fa795 100644 --- a/common/src/main/java/org/apache/atlas/annotation/ConditionalOnAtlasProperty.java +++ b/common/src/main/java/org/apache/atlas/annotation/ConditionalOnAtlasProperty.java @@ -25,12 +25,13 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -@Target({ ElementType.TYPE, ElementType.METHOD }) +@Target({ElementType.TYPE, ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) @Conditional(OnAtlasPropertyCondition.class) public @interface ConditionalOnAtlasProperty { // Configured atlas property String property(); + // The default interface implementation should declare this as true boolean isDefault() default false; } diff --git a/common/src/main/java/org/apache/atlas/annotation/Timed.java b/common/src/main/java/org/apache/atlas/annotation/Timed.java index b892a16c74..0403dcd94d 100644 --- a/common/src/main/java/org/apache/atlas/annotation/Timed.java +++ b/common/src/main/java/org/apache/atlas/annotation/Timed.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,16 +17,15 @@ */ package org.apache.atlas.annotation; - import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -@Target({ElementType.METHOD}) +@Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) @Inherited public @interface Timed { String value() default ""; -} \ No newline at end of file +} diff --git a/common/src/main/java/org/apache/atlas/groovy/AbstractFunctionExpression.java b/common/src/main/java/org/apache/atlas/groovy/AbstractFunctionExpression.java index 2e2307c0d9..185f79c75b 100644 --- a/common/src/main/java/org/apache/atlas/groovy/AbstractFunctionExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/AbstractFunctionExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,9 +21,8 @@ * Base class for all expression that can have a caller. */ public abstract class AbstractFunctionExpression extends AbstractGroovyExpression { - // null for global functions - private GroovyExpression caller; + private GroovyExpression caller; private TraversalStepType type = TraversalStepType.NONE; public AbstractFunctionExpression(GroovyExpression target) { @@ -32,10 +31,10 @@ public AbstractFunctionExpression(GroovyExpression target) { public AbstractFunctionExpression(TraversalStepType type, GroovyExpression target) { this.caller = target; - this.type = type; + this.type = type; } - public GroovyExpression getCaller() { + public GroovyExpression getCaller() { return caller; } @@ -43,15 +42,12 @@ public void setCaller(GroovyExpression expr) { caller = expr; } - - public void setType(TraversalStepType type) { - this.type = type; - } - @Override public TraversalStepType getType() { return type; } - + public void setType(TraversalStepType type) { + this.type = type; + } } diff --git a/common/src/main/java/org/apache/atlas/groovy/AbstractGroovyExpression.java b/common/src/main/java/org/apache/atlas/groovy/AbstractGroovyExpression.java index e4a7781094..327f75e8c7 100644 --- a/common/src/main/java/org/apache/atlas/groovy/AbstractGroovyExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/AbstractGroovyExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,6 @@ * */ public abstract class AbstractGroovyExpression implements GroovyExpression { - @Override public String toString() { GroovyGenerationContext ctx = new GroovyGenerationContext(); @@ -34,12 +33,12 @@ public String toString() { } @Override - public TraversalStepType getType() { - return TraversalStepType.NONE; + public GroovyExpression copy() { + return copy(getChildren()); } @Override - public GroovyExpression copy() { - return copy(getChildren()); + public TraversalStepType getType() { + return TraversalStepType.NONE; } } diff --git a/common/src/main/java/org/apache/atlas/groovy/ArithmeticExpression.java b/common/src/main/java/org/apache/atlas/groovy/ArithmeticExpression.java index fff104b7c8..715da801d1 100644 --- a/common/src/main/java/org/apache/atlas/groovy/ArithmeticExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/ArithmeticExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,47 +25,47 @@ * Represents an arithmetic expression such as a+b. */ public class ArithmeticExpression extends BinaryExpression { + public ArithmeticExpression(GroovyExpression left, ArithmeticOperator op, GroovyExpression right) { + super(left, op.getGroovyValue(), right); + } + + private ArithmeticExpression(GroovyExpression left, String op, GroovyExpression right) { + super(left, op, right); + } + + @Override + public GroovyExpression copy(List newChildren) { + return new ArithmeticExpression(newChildren.get(0), op, newChildren.get(1)); + } /** * Allowed arithmetic operators. */ - public enum ArithmeticOperator { + public enum ArithmeticOperator { PLUS("+"), MINUS("-"), TIMES("*"), DIVIDE("/"), MODULUS("%"); - private String groovyValue; + private final String groovyValue; + ArithmeticOperator(String groovyValue) { this.groovyValue = groovyValue; } - public String getGroovyValue() { - return groovyValue; - } public static ArithmeticOperator lookup(String groovyValue) throws AtlasException { - for(ArithmeticOperator op : ArithmeticOperator.values()) { + for (ArithmeticOperator op : ArithmeticOperator.values()) { if (op.getGroovyValue().equals(groovyValue)) { return op; } } + throw new AtlasException("Unknown Operator:" + groovyValue); } - } - - public ArithmeticExpression(GroovyExpression left, ArithmeticOperator op, GroovyExpression right) { - super(left, op.getGroovyValue(), right); - } - - private ArithmeticExpression(GroovyExpression left, String op, GroovyExpression right) { - super(left, op, right); - } - - @Override - public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 2; - return new ArithmeticExpression(newChildren.get(0), op, newChildren.get(1)); + public String getGroovyValue() { + return groovyValue; + } } } diff --git a/common/src/main/java/org/apache/atlas/groovy/BinaryExpression.java b/common/src/main/java/org/apache/atlas/groovy/BinaryExpression.java index 852845ea38..417c6321c0 100644 --- a/common/src/main/java/org/apache/atlas/groovy/BinaryExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/BinaryExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,25 +29,24 @@ * */ public abstract class BinaryExpression extends AbstractGroovyExpression { - - - private GroovyExpression left; - private GroovyExpression right; - protected String op; + protected String op; + private final GroovyExpression left; + private final GroovyExpression right; public BinaryExpression(GroovyExpression left, String op, GroovyExpression right) { - this.left = left; - this.op = op; + this.left = left; + this.op = op; this.right = right; } @Override public void generateGroovy(GroovyGenerationContext context) { - left.generateGroovy(context); + context.append(" "); context.append(op); context.append(" "); + right.generateGroovy(context); } diff --git a/common/src/main/java/org/apache/atlas/groovy/CastExpression.java b/common/src/main/java/org/apache/atlas/groovy/CastExpression.java index 808f327f99..c820ae587a 100644 --- a/common/src/main/java/org/apache/atlas/groovy/CastExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/CastExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,22 +25,22 @@ * Groovy expression that represents a cast. */ public class CastExpression extends AbstractGroovyExpression { - - private GroovyExpression expr; - private String className; + private final GroovyExpression expr; + private final String className; public CastExpression(GroovyExpression expr, String className) { - this.expr = expr; - this.className = className; + this.expr = expr; + this.className = className; } @Override public void generateGroovy(GroovyGenerationContext context) { - context.append("(("); context.append(className); context.append(")"); + expr.generateGroovy(context); + context.append(")"); } @@ -48,9 +48,9 @@ public void generateGroovy(GroovyGenerationContext context) { public List getChildren() { return Collections.singletonList(expr); } + @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 1; return new CastExpression(newChildren.get(0), className); } } diff --git a/common/src/main/java/org/apache/atlas/groovy/ClosureExpression.java b/common/src/main/java/org/apache/atlas/groovy/ClosureExpression.java index a5ca0b6fdf..a85c866480 100644 --- a/common/src/main/java/org/apache/atlas/groovy/ClosureExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/ClosureExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,34 +28,8 @@ * Groovy expression that represents a Groovy closure. */ public class ClosureExpression extends AbstractGroovyExpression { - - /** - * Variable declaration in a closure. - */ - public static class VariableDeclaration { - private String type; - private String varName; - - public VariableDeclaration(String type, String varName) { - super(); - this.type = type; - this.varName = varName; - } - - public VariableDeclaration(String varName) { - this.varName = varName; - } - - public void append(GroovyGenerationContext context) { - if (type != null) { - context.append(type); - context.append(" "); - } - context.append(varName); - } - } - private List vars = new ArrayList<>(); - private StatementListExpression body = new StatementListExpression(); + private final List vars = new ArrayList<>(); + private final StatementListExpression body = new StatementListExpression(); public ClosureExpression(String... varNames) { this(null, varNames); @@ -69,6 +43,7 @@ public ClosureExpression(List varNames, GroovyExpression initialStmt) { if (initialStmt != null) { this.body.addStatement(initialStmt); } + for (String varName : varNames) { vars.add(new VariableDeclaration(varName)); } @@ -78,6 +53,7 @@ public ClosureExpression(GroovyExpression initialStmt, List if (initialStmt != null) { this.body.addStatement(initialStmt); } + vars.addAll(varNames); } @@ -95,35 +71,66 @@ public void replaceStatement(int index, GroovyExpression newExpr) { @Override public void generateGroovy(GroovyGenerationContext context) { - context.append("{"); + if (!vars.isEmpty()) { - Iterator varIt = vars.iterator(); - while(varIt.hasNext()) { + for (Iterator varIt = vars.iterator(); varIt.hasNext(); ) { VariableDeclaration var = varIt.next(); + var.append(context); + if (varIt.hasNext()) { context.append(", "); } } + context.append("->"); } + body.generateGroovy(context); + context.append("}"); } @Override public List getChildren() { - return Collections.singletonList(body); + return Collections.singletonList(body); + } + + @Override + public GroovyExpression copy(List newChildren) { + return new ClosureExpression(newChildren.get(0), vars); } public List getStatements() { return body.getStatements(); } - @Override - public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 1; - return new ClosureExpression(newChildren.get(0), vars); + /** + * Variable declaration in a closure. + */ + public static class VariableDeclaration { + private final String type; + private final String varName; + + public VariableDeclaration(String type, String varName) { + super(); + + this.type = type; + this.varName = varName; + } + + public VariableDeclaration(String varName) { + this(null, varName); + } + + public void append(GroovyGenerationContext context) { + if (type != null) { + context.append(type); + context.append(" "); + } + + context.append(varName); + } } } diff --git a/common/src/main/java/org/apache/atlas/groovy/ComparisonExpression.java b/common/src/main/java/org/apache/atlas/groovy/ComparisonExpression.java index b64533f464..11bd83c1f1 100644 --- a/common/src/main/java/org/apache/atlas/groovy/ComparisonExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/ComparisonExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,23 +17,34 @@ */ package org.apache.atlas.groovy; -import java.util.List; - import org.apache.atlas.AtlasException; +import java.util.List; + /** * Represents an expression that compares two expressions * using a comparison operator (==, <, >, etc). * */ public class ComparisonExpression extends BinaryExpression { + public ComparisonExpression(GroovyExpression left, ComparisonOperator op, GroovyExpression right) { + super(left, op.getGroovyValue(), right); + } + + private ComparisonExpression(GroovyExpression left, String op, GroovyExpression right) { + super(left, op, right); + } + + @Override + public GroovyExpression copy(List newChildren) { + return new ComparisonExpression(newChildren.get(0), op, newChildren.get(1)); + } /** * Allowed comparison operators. * */ - public enum ComparisonOperator { - + public enum ComparisonOperator { GREATER_THAN_EQ_(">="), GREATER_THAN(">"), EQUALS("=="), @@ -41,36 +52,24 @@ public enum ComparisonOperator { LESS_THAN("<"), LESS_THAN_EQ("<="); - private String groovyValue; + private final String groovyValue; + ComparisonOperator(String groovyValue) { this.groovyValue = groovyValue; } - public String getGroovyValue() { - return groovyValue; - } public static ComparisonOperator lookup(String groovyValue) throws AtlasException { - for(ComparisonOperator op : ComparisonOperator.values()) { + for (ComparisonOperator op : ComparisonOperator.values()) { if (op.getGroovyValue().equals(groovyValue)) { return op; } } + throw new AtlasException("Unknown Operator:" + groovyValue); } - } - - public ComparisonExpression(GroovyExpression left, ComparisonOperator op, GroovyExpression right) { - super(left, op.getGroovyValue(), right); - } - - private ComparisonExpression(GroovyExpression left, String op, GroovyExpression right) { - super(left, op, right); - } - - @Override - public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 2; - return new ComparisonExpression(newChildren.get(0), op, newChildren.get(1)); + public String getGroovyValue() { + return groovyValue; + } } } diff --git a/common/src/main/java/org/apache/atlas/groovy/ComparisonOperatorExpression.java b/common/src/main/java/org/apache/atlas/groovy/ComparisonOperatorExpression.java index c9e363e112..7753b962d4 100644 --- a/common/src/main/java/org/apache/atlas/groovy/ComparisonOperatorExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/ComparisonOperatorExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,14 +27,12 @@ * */ public class ComparisonOperatorExpression extends BinaryExpression { - public ComparisonOperatorExpression(GroovyExpression left, GroovyExpression right) { super(left, "<=>", right); } @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 2; return new ComparisonOperatorExpression(newChildren.get(0), newChildren.get(1)); } } diff --git a/common/src/main/java/org/apache/atlas/groovy/FieldExpression.java b/common/src/main/java/org/apache/atlas/groovy/FieldExpression.java index 6a182ad6d3..e79ea7b056 100644 --- a/common/src/main/java/org/apache/atlas/groovy/FieldExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/FieldExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,19 +25,19 @@ * Groovy expression that accesses a field in an object. */ public class FieldExpression extends AbstractFunctionExpression { - - private String fieldName; + private final String fieldName; public FieldExpression(GroovyExpression target, String fieldName) { super(target); + this.fieldName = fieldName; } @Override public void generateGroovy(GroovyGenerationContext context) { getCaller().generateGroovy(context); - context.append(".'"); + context.append(".'"); context.append(fieldName); context.append("'"); } @@ -49,7 +49,6 @@ public List getChildren() { @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 1; return new FieldExpression(newChildren.get(0), fieldName); } } diff --git a/common/src/main/java/org/apache/atlas/groovy/FunctionCallExpression.java b/common/src/main/java/org/apache/atlas/groovy/FunctionCallExpression.java index ad09e3f962..1521cfa52a 100644 --- a/common/src/main/java/org/apache/atlas/groovy/FunctionCallExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/FunctionCallExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,45 +28,54 @@ * Groovy expression that calls a method on an object. */ public class FunctionCallExpression extends AbstractFunctionExpression { - - private String functionName; - private List arguments = new ArrayList<>(); + private final String functionName; + private final List arguments = new ArrayList<>(); public FunctionCallExpression(TraversalStepType type, String functionName, GroovyExpression... arguments) { super(type, null); + this.functionName = functionName; + this.arguments.addAll(Arrays.asList(arguments)); } public FunctionCallExpression(String functionName, GroovyExpression... arguments) { super(null); + this.functionName = functionName; + this.arguments.addAll(Arrays.asList(arguments)); } public FunctionCallExpression(TraversalStepType type, String functionName, List arguments) { super(type, null); + this.functionName = functionName; + this.arguments.addAll(arguments); } public FunctionCallExpression(GroovyExpression target, String functionName, GroovyExpression... arguments) { super(target); + this.functionName = functionName; + this.arguments.addAll(Arrays.asList(arguments)); } - public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName, - List arguments) { + public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName, List arguments) { super(type, target); + this.functionName = functionName; + this.arguments.addAll(arguments); } - public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName, - GroovyExpression... arguments) { + public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName, GroovyExpression... arguments) { super(type, target); + this.functionName = functionName; + this.arguments.addAll(Arrays.asList(arguments)); } @@ -78,63 +87,69 @@ public List getArguments() { return Collections.unmodifiableList(arguments); } - public String getFunctionName() { return functionName; } @Override public void generateGroovy(GroovyGenerationContext context) { - if (getCaller() != null) { getCaller().generateGroovy(context); context.append("."); } + context.append(functionName); context.append("("); - Iterator it = arguments.iterator(); - while (it.hasNext()) { + + for (Iterator it = arguments.iterator(); it.hasNext(); ) { GroovyExpression expr = it.next(); + expr.generateGroovy(context); + if (it.hasNext()) { context.append(","); } } + context.append(")"); } @Override public List getChildren() { List result = new ArrayList<>(arguments.size() + 1); + if (getCaller() != null) { result.add(getCaller()); } + result.addAll(arguments); + return result; } @Override public GroovyExpression copy(List newChildren) { - if (getCaller() == null) { return new FunctionCallExpression(getType(), functionName, newChildren); } - GroovyExpression newTarget = newChildren.get(0); - List args = null; + GroovyExpression newTarget = newChildren.get(0); + List args; + if (newChildren.size() > 1) { args = newChildren.subList(1, newChildren.size()); } else { args = Collections.emptyList(); } - return new FunctionCallExpression(getType(), newTarget, functionName, args); + return new FunctionCallExpression(getType(), newTarget, functionName, args); } public void setArgument(int index, GroovyExpression value) { if (index < 0 || index >= arguments.size()) { throw new IllegalArgumentException("Invalid argIndex " + index); } + arguments.set(index, value); } } diff --git a/common/src/main/java/org/apache/atlas/groovy/GroovyExpression.java b/common/src/main/java/org/apache/atlas/groovy/GroovyExpression.java index 839914729d..43d3a5e175 100644 --- a/common/src/main/java/org/apache/atlas/groovy/GroovyExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/GroovyExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,7 @@ * Represents an expression in the Groovy programming language, which * is the language that Gremlin scripts are written and interpreted in. */ -public interface GroovyExpression { +public interface GroovyExpression { /** * Generates a Groovy script from the expression. * diff --git a/common/src/main/java/org/apache/atlas/groovy/GroovyGenerationContext.java b/common/src/main/java/org/apache/atlas/groovy/GroovyGenerationContext.java index a4e485b704..023f95d634 100644 --- a/common/src/main/java/org/apache/atlas/groovy/GroovyGenerationContext.java +++ b/common/src/main/java/org/apache/atlas/groovy/GroovyGenerationContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,13 +29,12 @@ * */ public class GroovyGenerationContext { - - private boolean parametersAllowed = true; - private int parameterCount = 0; - private Map parameterValues = new HashMap<>(); + private boolean parametersAllowed = true; + private final Map parameterValues = new HashMap<>(); + private int parameterCount; //used to build up the groovy script - private StringBuilder generatedQuery = new StringBuilder(); + private final StringBuilder generatedQuery = new StringBuilder(); public void setParametersAllowed(boolean value) { this.parametersAllowed = value; @@ -44,17 +43,22 @@ public void setParametersAllowed(boolean value) { public GroovyExpression addParameter(Object value) { if (this.parametersAllowed) { String parameterName = "p" + (++parameterCount); + this.parameterValues.put(parameterName, value); + return new IdentifierExpression(parameterName); } else { LiteralExpression expr = new LiteralExpression(value); + expr.setTranslateToParameter(false); + return expr; } } public StringBuilder append(String gremlin) { this.generatedQuery.append(gremlin); + return generatedQuery; } diff --git a/common/src/main/java/org/apache/atlas/groovy/IdentifierExpression.java b/common/src/main/java/org/apache/atlas/groovy/IdentifierExpression.java index 4c0694a407..a4c256a3b2 100644 --- a/common/src/main/java/org/apache/atlas/groovy/IdentifierExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/IdentifierExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,20 +26,18 @@ * */ public class IdentifierExpression extends AbstractGroovyExpression { - - private TraversalStepType type = TraversalStepType.NONE; - private String varName; + private TraversalStepType type = TraversalStepType.NONE; + private final String varName; public IdentifierExpression(String varName) { this.varName = varName; } public IdentifierExpression(TraversalStepType type, String varName) { + this.type = type; this.varName = varName; - this.type = type; } - public String getVariableName() { return varName; } @@ -56,18 +54,15 @@ public List getChildren() { @Override public GroovyExpression copy(List newChildren) { - assert newChildren.isEmpty(); - IdentifierExpression result = new IdentifierExpression(varName); - result.setType(type); - return result; - } - - public void setType(TraversalStepType type) { - this.type = type; + return new IdentifierExpression(type, varName); } @Override public TraversalStepType getType() { return type; } + + public void setType(TraversalStepType type) { + this.type = type; + } } diff --git a/common/src/main/java/org/apache/atlas/groovy/LabeledExpression.java b/common/src/main/java/org/apache/atlas/groovy/LabeledExpression.java index a99341024c..e58a68811f 100644 --- a/common/src/main/java/org/apache/atlas/groovy/LabeledExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/LabeledExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,19 +25,19 @@ * Represents a Groovy expression that has a label. */ public class LabeledExpression extends AbstractGroovyExpression { - - private String label; - private GroovyExpression expr; + private final String label; + private final GroovyExpression expr; public LabeledExpression(String label, GroovyExpression expr) { this.label = label; - this.expr = expr; + this.expr = expr; } @Override public void generateGroovy(GroovyGenerationContext context) { context.append(label); context.append(":"); + expr.generateGroovy(context); } @@ -48,7 +48,6 @@ public List getChildren() { @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 1; return new LabeledExpression(label, newChildren.get(0)); } } diff --git a/common/src/main/java/org/apache/atlas/groovy/ListExpression.java b/common/src/main/java/org/apache/atlas/groovy/ListExpression.java index 7969426011..873c3d9ab5 100644 --- a/common/src/main/java/org/apache/atlas/groovy/ListExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/ListExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,10 +28,8 @@ * Groovy expression that represents a list literal. */ public class ListExpression extends AbstractGroovyExpression { - private List values = new ArrayList<>(); - public ListExpression(GroovyExpression... values) { this.values = Arrays.asList(values); } @@ -45,16 +43,18 @@ public void addArgument(GroovyExpression expr) { } public void generateGroovy(GroovyGenerationContext context) { - context.append("["); - Iterator it = values.iterator(); - while(it.hasNext()) { + + for (Iterator it = values.iterator(); it.hasNext(); ) { GroovyExpression expr = it.next(); + expr.generateGroovy(context); + if (it.hasNext()) { context.append(", "); } } + context.append("]"); } @@ -67,6 +67,4 @@ public List getChildren() { public GroovyExpression copy(List newChildren) { return new ListExpression(newChildren); } - - } diff --git a/common/src/main/java/org/apache/atlas/groovy/LiteralExpression.java b/common/src/main/java/org/apache/atlas/groovy/LiteralExpression.java index 14074994f7..c55f25e85f 100644 --- a/common/src/main/java/org/apache/atlas/groovy/LiteralExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/LiteralExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,96 +27,97 @@ * Represents a literal value. */ public class LiteralExpression extends AbstractGroovyExpression { - - public static final LiteralExpression TRUE = new LiteralExpression(true); + public static final LiteralExpression TRUE = new LiteralExpression(true); public static final LiteralExpression FALSE = new LiteralExpression(false); - public static final LiteralExpression NULL = new LiteralExpression(null); + public static final LiteralExpression NULL = new LiteralExpression(null); - private Object value; - private boolean translateToParameter = true; - private boolean addTypeSuffix = false; + private final Object value; + private boolean translateToParameter; + private boolean addTypeSuffix; public LiteralExpression(Object value, boolean addTypeSuffix) { - this.value = value; + this.value = value; this.translateToParameter = value instanceof String; - this.addTypeSuffix = addTypeSuffix; + this.addTypeSuffix = addTypeSuffix; } public LiteralExpression(Object value, boolean addTypeSuffix, boolean translateToParameter) { - this.value = value; + this.value = value; this.translateToParameter = translateToParameter; - this.addTypeSuffix = addTypeSuffix; + this.addTypeSuffix = addTypeSuffix; } public LiteralExpression(Object value) { - this.value = value; + this.value = value; this.translateToParameter = value instanceof String; } - private String getTypeSuffix() { - if (!addTypeSuffix) { - return ""; - } - if (value.getClass() == Long.class) { - return "L"; - } - - if (value.getClass() == Float.class) { - return "f"; - } - - if (value.getClass() == Double.class) { - return "d"; - } - - return ""; - } - @Override public void generateGroovy(GroovyGenerationContext context) { - if (translateToParameter) { GroovyExpression result = context.addParameter(value); + result.generateGroovy(context); + return; } if (value instanceof String) { String escapedValue = getEscapedValue(); + context.append("'"); context.append(escapedValue); context.append("'"); - } else { context.append(String.valueOf(value)); context.append(getTypeSuffix()); } + } + @Override + public List getChildren() { + return Collections.emptyList(); } - public Object getValue() { - return value; + @Override + public GroovyExpression copy(List newChildren) { + return new LiteralExpression(value, addTypeSuffix, translateToParameter); } - private String getEscapedValue() { - String escapedValue = (String)value; - escapedValue = escapedValue.replaceAll(Pattern.quote("\\"), Matcher.quoteReplacement("\\\\")); - escapedValue = escapedValue.replaceAll(Pattern.quote("'"), Matcher.quoteReplacement("\\'")); - return escapedValue; + public Object getValue() { + return value; } public void setTranslateToParameter(boolean translateToParameter) { this.translateToParameter = translateToParameter; } - @Override - public List getChildren() { - return Collections.emptyList(); + private String getTypeSuffix() { + if (!addTypeSuffix) { + return ""; + } + + if (value.getClass() == Long.class) { + return "L"; + } + + if (value.getClass() == Float.class) { + return "f"; + } + + if (value.getClass() == Double.class) { + return "d"; + } + + return ""; } - @Override - public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 0; - return new LiteralExpression(value, addTypeSuffix, translateToParameter); + private String getEscapedValue() { + String escapedValue = (String) value; + + escapedValue = escapedValue.replaceAll(Pattern.quote("\\"), Matcher.quoteReplacement("\\\\")); + escapedValue = escapedValue.replaceAll(Pattern.quote("'"), Matcher.quoteReplacement("\\'")); + + return escapedValue; } } diff --git a/common/src/main/java/org/apache/atlas/groovy/LogicalExpression.java b/common/src/main/java/org/apache/atlas/groovy/LogicalExpression.java index 68e6847a03..0f50154ed0 100644 --- a/common/src/main/java/org/apache/atlas/groovy/LogicalExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/LogicalExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,35 +24,35 @@ * */ public class LogicalExpression extends BinaryExpression { + public LogicalExpression(GroovyExpression left, LogicalOperator op, GroovyExpression right) { + super(left, op.getGroovyValue(), right); + } + + private LogicalExpression(GroovyExpression left, String op, GroovyExpression right) { + super(left, op, right); + } + + @Override + public GroovyExpression copy(List newChildren) { + return new LogicalExpression(newChildren.get(0), op, newChildren.get(1)); + } /** * Allowed logical operators. * */ - public enum LogicalOperator { + public enum LogicalOperator { AND("&&"), OR("||"); - private String groovyValue; + private final String groovyValue; + LogicalOperator(String groovyValue) { this.groovyValue = groovyValue; } + public String getGroovyValue() { return groovyValue; } } - - public LogicalExpression(GroovyExpression left, LogicalOperator op, GroovyExpression right) { - super(left, op.getGroovyValue(), right); - } - - private LogicalExpression(GroovyExpression left, String op, GroovyExpression right) { - super(left, op, right); - } - - @Override - public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 2; - return new LogicalExpression(newChildren.get(0), op, newChildren.get(1)); - } } diff --git a/common/src/main/java/org/apache/atlas/groovy/RangeExpression.java b/common/src/main/java/org/apache/atlas/groovy/RangeExpression.java index 977adb6b03..be449dbd09 100644 --- a/common/src/main/java/org/apache/atlas/groovy/RangeExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/RangeExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,21 +25,22 @@ * Represents an "exclusive" range expression, e.g. [0..<10]. */ public class RangeExpression extends AbstractFunctionExpression { - - private TraversalStepType stepType; - private int startIndex; - private int endIndex; + private final TraversalStepType stepType; + private int startIndex; + private int endIndex; public RangeExpression(TraversalStepType stepType, GroovyExpression parent, int offset, int count) { super(parent); + this.startIndex = offset; - this.endIndex = count; - this.stepType = stepType; + this.endIndex = count; + this.stepType = stepType; } @Override public void generateGroovy(GroovyGenerationContext context) { getCaller().generateGroovy(context); + context.append(" ["); new LiteralExpression(startIndex).generateGroovy(context); context.append("..<"); @@ -54,7 +55,6 @@ public List getChildren() { @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 1; return new RangeExpression(stepType, newChildren.get(0), startIndex, endIndex); } @@ -64,22 +64,18 @@ public TraversalStepType getType() { } public int getStartIndex() { - return startIndex; } public void setStartIndex(int startIndex) { - this.startIndex = startIndex; } public int getEndIndex() { - return endIndex; } public void setEndIndex(int endIndex) { - this.endIndex = endIndex; } } diff --git a/common/src/main/java/org/apache/atlas/groovy/StatementListExpression.java b/common/src/main/java/org/apache/atlas/groovy/StatementListExpression.java index f9c88ec116..2d10fa41f3 100644 --- a/common/src/main/java/org/apache/atlas/groovy/StatementListExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/StatementListExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,11 +27,9 @@ * Represents a semi-colon delimited list of Groovy expressions. */ public class StatementListExpression extends AbstractGroovyExpression { - - private List stmts = new ArrayList<>(); + private final List stmts = new ArrayList<>(); public StatementListExpression() { - } /** @@ -43,36 +41,31 @@ public StatementListExpression(List newChildren) { public void addStatement(GroovyExpression expr) { if (expr instanceof StatementListExpression) { - stmts.addAll(((StatementListExpression)expr).getStatements()); + stmts.addAll(((StatementListExpression) expr).getStatements()); } else { stmts.add(expr); } } public void addStatements(List exprs) { - for(GroovyExpression expr : exprs) { + for (GroovyExpression expr : exprs) { addStatement(expr); } } @Override public void generateGroovy(GroovyGenerationContext context) { - - Iterator stmtIt = stmts.iterator(); - while(stmtIt.hasNext()) { + for (Iterator stmtIt = stmts.iterator(); stmtIt.hasNext(); ) { GroovyExpression stmt = stmtIt.next(); + stmt.generateGroovy(context); + if (stmtIt.hasNext()) { context.append(";"); } } } - - public List getStatements() { - return stmts; - } - @Override public List getChildren() { return Collections.unmodifiableList(stmts); @@ -83,13 +76,17 @@ public GroovyExpression copy(List newChildren) { return new StatementListExpression(newChildren); } + public List getStatements() { + return stmts; + } + @Override public TraversalStepType getType() { return TraversalStepType.NONE; } /** - * @param oldExpr + * @param index * @param newExpr */ public void replaceStatement(int index, GroovyExpression newExpr) { diff --git a/common/src/main/java/org/apache/atlas/groovy/TernaryOperatorExpression.java b/common/src/main/java/org/apache/atlas/groovy/TernaryOperatorExpression.java index 8835dd2f31..3b01771294 100644 --- a/common/src/main/java/org/apache/atlas/groovy/TernaryOperatorExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/TernaryOperatorExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,22 +26,18 @@ * falseValue) */ public class TernaryOperatorExpression extends AbstractGroovyExpression { + private final GroovyExpression booleanExpr; + private final GroovyExpression trueValue; + private final GroovyExpression falseValue; - private GroovyExpression booleanExpr; - private GroovyExpression trueValue; - private GroovyExpression falseValue; - - public TernaryOperatorExpression(GroovyExpression booleanExpr, GroovyExpression trueValue, - GroovyExpression falseValue) { - + public TernaryOperatorExpression(GroovyExpression booleanExpr, GroovyExpression trueValue, GroovyExpression falseValue) { this.booleanExpr = booleanExpr; - this.trueValue = trueValue; - this.falseValue = falseValue; + this.trueValue = trueValue; + this.falseValue = falseValue; } @Override public void generateGroovy(GroovyGenerationContext context) { - context.append("(("); booleanExpr.generateGroovy(context); context.append(")?("); @@ -51,12 +47,6 @@ public void generateGroovy(GroovyGenerationContext context) { context.append("))"); } - public String toString() { - GroovyGenerationContext context = new GroovyGenerationContext(); - generateGroovy(context); - return context.getQuery(); - } - @Override public List getChildren() { return Arrays.asList(booleanExpr, trueValue, falseValue); @@ -64,10 +54,17 @@ public List getChildren() { @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 3; return new TernaryOperatorExpression(newChildren.get(0), newChildren.get(1), newChildren.get(2)); } + public String toString() { + GroovyGenerationContext context = new GroovyGenerationContext(); + + generateGroovy(context); + + return context.getQuery(); + } + @Override public TraversalStepType getType() { return trueValue.getType(); diff --git a/common/src/main/java/org/apache/atlas/groovy/TraversalStepType.java b/common/src/main/java/org/apache/atlas/groovy/TraversalStepType.java index fde88147c7..bf39b2bd8b 100644 --- a/common/src/main/java/org/apache/atlas/groovy/TraversalStepType.java +++ b/common/src/main/java/org/apache/atlas/groovy/TraversalStepType.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/src/main/java/org/apache/atlas/groovy/TypeCoersionExpression.java b/common/src/main/java/org/apache/atlas/groovy/TypeCoersionExpression.java index 956dafa0d0..94a14be33d 100644 --- a/common/src/main/java/org/apache/atlas/groovy/TypeCoersionExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/TypeCoersionExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,18 +25,16 @@ * Groovy expression that represents a type coersion (e.g obj as Set). */ public class TypeCoersionExpression extends AbstractGroovyExpression { - - private GroovyExpression expr; - private String className; + private final GroovyExpression expr; + private final String className; public TypeCoersionExpression(GroovyExpression expr, String className) { - this.expr = expr; - this.className = className; + this.expr = expr; + this.className = className; } @Override public void generateGroovy(GroovyGenerationContext context) { - context.append("(("); expr.generateGroovy(context); context.append(")"); @@ -52,8 +50,6 @@ public List getChildren() { @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 1; return new TypeCoersionExpression(newChildren.get(0), className); } - } diff --git a/common/src/main/java/org/apache/atlas/groovy/VariableAssignmentExpression.java b/common/src/main/java/org/apache/atlas/groovy/VariableAssignmentExpression.java index 1aa74435d5..6ef656b862 100644 --- a/common/src/main/java/org/apache/atlas/groovy/VariableAssignmentExpression.java +++ b/common/src/main/java/org/apache/atlas/groovy/VariableAssignmentExpression.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,18 +25,18 @@ * Groovy statement that assigns a value to a variable. */ public class VariableAssignmentExpression extends AbstractGroovyExpression { - - private String type = null; - private String name; - private GroovyExpression value; + private final String type; + private final String name; + private final GroovyExpression value; /** - * @param string + * @param type + * @param name * @param v */ public VariableAssignmentExpression(String type, String name, GroovyExpression v) { - this.type = type; - this.name = name; + this.type = type; + this.name = name; this.value = v; } @@ -52,10 +52,11 @@ public void generateGroovy(GroovyGenerationContext context) { context.append(type); context.append(" "); } + context.append(name); context.append("="); - value.generateGroovy(context); + value.generateGroovy(context); } @Override @@ -65,8 +66,6 @@ public List getChildren() { @Override public GroovyExpression copy(List newChildren) { - assert newChildren.size() == 1; return new VariableAssignmentExpression(name, newChildren.get(0)); } - } diff --git a/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java b/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java index 91fd4e1076..c75b9ebe01 100644 --- a/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java +++ b/common/src/main/java/org/apache/atlas/ha/HAConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,38 +30,34 @@ * A wrapper for getting configuration entries related to HighAvailability. */ public final class HAConfiguration { - - - public static final String ATLAS_SERVER_ZK_ROOT_DEFAULT = "/apache_atlas"; + public static final String ATLAS_SERVER_ZK_ROOT_DEFAULT = "/apache_atlas"; + public static final String ATLAS_SERVER_HA_PREFIX = "atlas.server.ha."; + public static final String ZOOKEEPER_PREFIX = "zookeeper."; + public static final String ATLAS_SERVER_HA_ZK_ROOT_KEY = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "zkroot"; + public static final String ATLAS_SERVER_HA_ENABLED_KEY = ATLAS_SERVER_HA_PREFIX + "enabled"; + public static final String ATLAS_SERVER_ADDRESS_PREFIX = "atlas.server.address."; + public static final String ATLAS_SERVER_IDS = "atlas.server.ids"; + public static final String HA_ZOOKEEPER_CONNECT = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "connect"; + public static final int DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS = 1000; + public static final String HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "retry.sleeptime.ms"; + public static final String HA_ZOOKEEPER_NUM_RETRIES = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "num.retries"; + public static final int DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES = 3; + public static final String HA_ZOOKEEPER_SESSION_TIMEOUT_MS = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "session.timeout.ms"; + public static final int DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS = 20000; + public static final String HA_ZOOKEEPER_ACL = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "acl"; + public static final String HA_ZOOKEEPER_AUTH = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "auth"; private HAConfiguration() { + // to block instantiation } - public static final String ATLAS_SERVER_HA_PREFIX = "atlas.server.ha."; - public static final String ZOOKEEPER_PREFIX = "zookeeper."; - public static final String ATLAS_SERVER_HA_ZK_ROOT_KEY = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "zkroot"; - public static final String ATLAS_SERVER_HA_ENABLED_KEY = ATLAS_SERVER_HA_PREFIX + "enabled"; - public static final String ATLAS_SERVER_ADDRESS_PREFIX = "atlas.server.address."; - public static final String ATLAS_SERVER_IDS = "atlas.server.ids"; - public static final String HA_ZOOKEEPER_CONNECT = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "connect"; - public static final int DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS = 1000; - public static final String HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS = - ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "retry.sleeptime.ms"; - public static final String HA_ZOOKEEPER_NUM_RETRIES = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "num.retries"; - public static final int DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES = 3; - public static final String HA_ZOOKEEPER_SESSION_TIMEOUT_MS = - ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "session.timeout.ms"; - public static final int DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS = 20000; - public static final String HA_ZOOKEEPER_ACL = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "acl"; - public static final String HA_ZOOKEEPER_AUTH = ATLAS_SERVER_HA_PREFIX + ZOOKEEPER_PREFIX + "auth"; - /** * Return whether HA is enabled or not. * @param configuration underlying configuration instance * @return */ public static boolean isHAEnabled(Configuration configuration) { - boolean ret = false; + boolean ret; if (configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)) { ret = configuration.getBoolean(ATLAS_SERVER_HA_ENABLED_KEY); @@ -85,42 +81,63 @@ public static boolean isHAEnabled(Configuration configuration) { * @return */ public static String getBoundAddressForId(Configuration configuration, String serverId) { - String hostPort = configuration.getString(ATLAS_SERVER_ADDRESS_PREFIX +serverId); + String hostPort = configuration.getString(ATLAS_SERVER_ADDRESS_PREFIX + serverId); boolean isSecure = configuration.getBoolean(SecurityProperties.TLS_ENABLED); - String protocol = (isSecure) ? "https://" : "http://"; + String protocol = (isSecure) ? "https://" : "http://"; + return protocol + hostPort; } public static List getServerInstances(Configuration configuration) { - String[] serverIds = configuration.getStringArray(ATLAS_SERVER_IDS); + String[] serverIds = configuration.getStringArray(ATLAS_SERVER_IDS); List serverInstances = new ArrayList<>(serverIds.length); + for (String serverId : serverIds) { serverInstances.add(getBoundAddressForId(configuration, serverId)); } + return serverInstances; } + public static ZookeeperProperties getZookeeperProperties(Configuration configuration) { + String[] zkServers; + + if (configuration.containsKey(HA_ZOOKEEPER_CONNECT)) { + zkServers = configuration.getStringArray(HA_ZOOKEEPER_CONNECT); + } else { + zkServers = configuration.getStringArray("atlas.kafka." + ZOOKEEPER_PREFIX + "connect"); + } + + String zkRoot = configuration.getString(ATLAS_SERVER_HA_ZK_ROOT_KEY, ATLAS_SERVER_ZK_ROOT_DEFAULT); + int retriesSleepTimeMillis = configuration.getInt(HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS, DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS); + int numRetries = configuration.getInt(HA_ZOOKEEPER_NUM_RETRIES, DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES); + int sessionTimeout = configuration.getInt(HA_ZOOKEEPER_SESSION_TIMEOUT_MS, DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS); + String acl = configuration.getString(HA_ZOOKEEPER_ACL); + String auth = configuration.getString(HA_ZOOKEEPER_AUTH); + + return new ZookeeperProperties(StringUtils.join(zkServers, ','), zkRoot, retriesSleepTimeMillis, numRetries, sessionTimeout, acl, auth); + } + /** * A collection of Zookeeper specific configuration that is used by High Availability code. */ public static class ZookeeperProperties { - private String connectString; - private String zkRoot; - private int retriesSleepTimeMillis; - private int numRetries; - private int sessionTimeout; - private String acl; - private String auth; - - public ZookeeperProperties(String connectString, String zkRoot, int retriesSleepTimeMillis, int numRetries, - int sessionTimeout, String acl, String auth) { - this.connectString = connectString; - this.zkRoot = zkRoot; + private final String connectString; + private final String zkRoot; + private final int retriesSleepTimeMillis; + private final int numRetries; + private final int sessionTimeout; + private final String acl; + private final String auth; + + public ZookeeperProperties(String connectString, String zkRoot, int retriesSleepTimeMillis, int numRetries, int sessionTimeout, String acl, String auth) { + this.connectString = connectString; + this.zkRoot = zkRoot; this.retriesSleepTimeMillis = retriesSleepTimeMillis; - this.numRetries = numRetries; - this.sessionTimeout = sessionTimeout; - this.acl = acl; - this.auth = auth; + this.numRetries = numRetries; + this.sessionTimeout = sessionTimeout; + this.acl = acl; + this.auth = auth; } public String getConnectString() { @@ -151,11 +168,21 @@ public String getZkRoot() { return zkRoot; } + @Override + public int hashCode() { + return Objects.hash(connectString, zkRoot, retriesSleepTimeMillis, numRetries, sessionTimeout, acl, auth); + } + @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } else if (o == null || getClass() != o.getClass()) { + return false; + } + ZookeeperProperties that = (ZookeeperProperties) o; + return retriesSleepTimeMillis == that.retriesSleepTimeMillis && numRetries == that.numRetries && sessionTimeout == that.sessionTimeout && @@ -165,43 +192,12 @@ public boolean equals(Object o) { Objects.equals(auth, that.auth); } - @Override - public int hashCode() { - return Objects.hash(connectString, zkRoot, retriesSleepTimeMillis, numRetries, sessionTimeout, acl, auth); - } - public boolean hasAcl() { - return getAcl()!=null; + return getAcl() != null; } public boolean hasAuth() { - return getAuth()!=null; - } - } - - public static ZookeeperProperties getZookeeperProperties(Configuration configuration) { - String[] zkServers; - if (configuration.containsKey(HA_ZOOKEEPER_CONNECT)) { - zkServers = configuration.getStringArray(HA_ZOOKEEPER_CONNECT); - } else { - zkServers = configuration.getStringArray("atlas.kafka." + ZOOKEEPER_PREFIX + "connect"); + return getAuth() != null; } - - String zkRoot = configuration.getString(ATLAS_SERVER_HA_ZK_ROOT_KEY, ATLAS_SERVER_ZK_ROOT_DEFAULT); - int retriesSleepTimeMillis = configuration.getInt(HA_ZOOKEEPER_RETRY_SLEEPTIME_MILLIS, - DEFAULT_ZOOKEEPER_CONNECT_SLEEPTIME_MILLIS); - - int numRetries = configuration.getInt(HA_ZOOKEEPER_NUM_RETRIES, DEFAULT_ZOOKEEPER_CONNECT_NUM_RETRIES); - - int sessionTimeout = configuration.getInt(HA_ZOOKEEPER_SESSION_TIMEOUT_MS, - DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MILLIS); - - String acl = configuration.getString(HA_ZOOKEEPER_ACL); - String auth = configuration.getString(HA_ZOOKEEPER_AUTH); - - return new ZookeeperProperties(StringUtils.join(zkServers, ','), - zkRoot, - retriesSleepTimeMillis, numRetries, - sessionTimeout, acl, auth); } } diff --git a/common/src/main/java/org/apache/atlas/pc/StatusReporter.java b/common/src/main/java/org/apache/atlas/pc/StatusReporter.java index 507e842b82..147fd4e13c 100644 --- a/common/src/main/java/org/apache/atlas/pc/StatusReporter.java +++ b/common/src/main/java/org/apache/atlas/pc/StatusReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,10 +30,10 @@ public class StatusReporter { private static final Logger LOG = LoggerFactory.getLogger(StatusReporter.class); - private Map producedItems = new LinkedHashMap<>(); - private Set processedSet = new HashSet<>(); - private long timeoutDuration; - private long lastAck; + private final Map producedItems = new LinkedHashMap<>(); + private final Set processedSet = new HashSet<>(); + private final long timeoutDuration; + private long lastAck; public StatusReporter() { this.timeoutDuration = -1; @@ -58,17 +58,29 @@ public void processed(T[] index) { public U ack() { U ack = null; U ret; + do { Map.Entry firstElement = getFirstElement(this.producedItems); + ret = completionIndex(firstElement); + if (ret != null) { ack = ret; } - } while(ret != null); + } + while (ret != null); return ack; } + public int getProducedCount() { + return this.producedItems.size(); + } + + public int getProcessedCount() { + return this.processedSet.size(); + } + private Map.Entry getFirstElement(Map map) { if (map.isEmpty()) { return null; @@ -79,12 +91,14 @@ private Map.Entry getFirstElement(Map map) { private U completionIndex(Map.Entry lookFor) { U ack = null; + if (lookFor == null) { return ack; } if (hasTimeoutDurationReached(System.currentTimeMillis())) { LOG.warn("Ack: Timeout: {} - {}", lookFor.getKey(), lookFor.getValue()); + return acknowledged(lookFor); } @@ -97,22 +111,18 @@ private U completionIndex(Map.Entry lookFor) { private U acknowledged(Map.Entry lookFor) { U ack = lookFor.getValue(); + producedItems.remove(lookFor.getKey()); processedSet.remove(lookFor.getKey()); + return ack; } private boolean hasTimeoutDurationReached(long now) { boolean b = (this.timeoutDuration > -1) && (this.lastAck != 0) && ((now - this.lastAck) >= timeoutDuration); - lastAck = System.currentTimeMillis(); - return b; - } - public int getProducedCount() { - return this.producedItems.size(); - } + lastAck = System.currentTimeMillis(); - public int getProcessedCount() { - return this.processedSet.size(); + return b; } } diff --git a/common/src/main/java/org/apache/atlas/pc/WorkItemBuilder.java b/common/src/main/java/org/apache/atlas/pc/WorkItemBuilder.java index 6dcb29d6af..c3b6fd7b90 100644 --- a/common/src/main/java/org/apache/atlas/pc/WorkItemBuilder.java +++ b/common/src/main/java/org/apache/atlas/pc/WorkItemBuilder.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java b/common/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java index dd76697a5c..2bcc4bab52 100644 --- a/common/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java +++ b/common/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java @@ -37,8 +37,8 @@ public abstract class WorkItemConsumer implements Runnable { private final BlockingQueue queue; private final AtomicBoolean isDirty = new AtomicBoolean(false); private final AtomicLong maxCommitTimeInMs = new AtomicLong(DEFAULT_COMMIT_TIME_IN_MS); - private CountDownLatch countdownLatch; - private Queue results; + private CountDownLatch countdownLatch; + private Queue results; public WorkItemConsumer(BlockingQueue queue) { this.queue = queue; @@ -76,6 +76,14 @@ public long getMaxCommitTimeInMs() { return ((commitTime > DEFAULT_COMMIT_TIME_IN_MS) ? commitTime : DEFAULT_COMMIT_TIME_IN_MS); } + public void setCountDownLatch(CountDownLatch countdownLatch) { + this.countdownLatch = countdownLatch; + } + + public void setResults(Queue queue) { + this.results = queue; + } + protected void commitDirty() { if (!isDirty.get()) { return; @@ -110,12 +118,4 @@ protected void updateCommitTime(long commitTime) { this.maxCommitTimeInMs.set(commitTime); } } - - public void setCountDownLatch(CountDownLatch countdownLatch) { - this.countdownLatch = countdownLatch; - } - - public void setResults(Queue queue) { - this.results = queue; - } } diff --git a/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java b/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java index 351421e8ef..c1f79e417e 100644 --- a/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java +++ b/common/src/main/java/org/apache/atlas/pc/WorkItemManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,7 @@ */ package org.apache.atlas.pc; + import org.apache.curator.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,19 +25,25 @@ import java.util.ArrayList; import java.util.List; import java.util.Queue; -import java.util.concurrent.*; - -public class WorkItemManager { +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +public class WorkItemManager> { private static final Logger LOG = LoggerFactory.getLogger(WorkItemManager.class); private final int numWorkers; private final BlockingQueue workQueue; private final ExecutorService service; private final List consumers = new ArrayList<>(); - private CountDownLatch countdownLatch; - private Queue resultsQueue; + private CountDownLatch countdownLatch; + private Queue resultsQueue; - public WorkItemManager(WorkItemBuilder builder, String namePrefix, int batchSize, int numWorkers, boolean collectResults) { + public WorkItemManager(WorkItemBuilder builder, String namePrefix, int batchSize, int numWorkers, boolean collectResults) { this.numWorkers = numWorkers; this.workQueue = new LinkedBlockingQueue<>(batchSize * numWorkers); this.service = Executors.newFixedThreadPool(numWorkers, new ThreadFactoryBuilder().setNameFormat(namePrefix + "-%d").build()); @@ -46,7 +53,7 @@ public WorkItemManager(WorkItemBuilder builder, String namePrefix, int batchSize start(); } - public WorkItemManager(WorkItemBuilder builder, int batchSize, int numWorkers) { + public WorkItemManager(WorkItemBuilder builder, int batchSize, int numWorkers) { this(builder, "workItemConsumer", batchSize, numWorkers, false); } @@ -54,22 +61,6 @@ public void setResultsCollection(Queue resultsQueue) { this.resultsQueue = resultsQueue; } - private void createConsumers(WorkItemBuilder builder, int numWorkers, boolean collectResults) { - if (collectResults) { - setResultsCollection(new ConcurrentLinkedQueue<>()); - } - - for (int i = 0; i < numWorkers; i++) { - U c = (U) builder.build(workQueue); - - consumers.add(c); - - if (collectResults) { - c.setResults(resultsQueue); - } - } - } - public void start() { this.countdownLatch = new CountDownLatch(numWorkers); @@ -107,7 +98,9 @@ public void drain() { } LOG.debug("Drain: Stated! Queue size: {}", workQueue.size()); + this.countdownLatch.await(); + LOG.debug("Drain: Done! Queue size: {}", workQueue.size()); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); @@ -125,17 +118,33 @@ public void shutdown() throws InterruptedException { LOG.info("WorkItemManager: Shutdown done!"); } - public Queue getResults() { + public Queue getResults() { return this.resultsQueue; } + private void createConsumers(WorkItemBuilder builder, int numWorkers, boolean collectResults) { + if (collectResults) { + setResultsCollection(new ConcurrentLinkedQueue<>()); + } + + for (int i = 0; i < numWorkers; i++) { + U c = builder.build(workQueue); + + consumers.add(c); + + if (collectResults) { + c.setResults(resultsQueue); + } + } + } + private int getAvgCommitTimeSeconds() { - int commitTimeSeconds = 0; + long commitTimeSeconds = 0; for (U c : consumers) { commitTimeSeconds += c.getMaxCommitTimeInMs(); } - return (commitTimeSeconds / consumers.size()) / 1000; + return (int) ((commitTimeSeconds / consumers.size()) / 1000); } } diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 2134133f5f..3addbcd0d1 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -26,10 +26,8 @@ /** * Repository Constants. - * */ public final class Constants { - /** * Globally Unique identifier property key. */ @@ -45,12 +43,12 @@ public final class Constants { /** * Entity type name property key. */ - public static final String ENTITY_TYPE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName"); + public static final String ENTITY_TYPE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName"); public static final String RELATIONSHIP_TYPE_PROPERTY_KEY = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + ENTITY_TYPE_PROPERTY_KEY); - public static final String TYPE_NAME_INTERNAL = INTERNAL_PROPERTY_KEY_PREFIX + "internal"; - public static final String ASSET_ENTITY_TYPE = "Asset"; - public static final String OWNER_ATTRIBUTE = "owner"; + public static final String TYPE_NAME_INTERNAL = INTERNAL_PROPERTY_KEY_PREFIX + "internal"; + public static final String ASSET_ENTITY_TYPE = "Asset"; + public static final String OWNER_ATTRIBUTE = "owner"; /** * Entity type's super types property key. @@ -84,20 +82,20 @@ public final class Constants { /** * Trait names property key and index name. */ - public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames"); - public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames"); - - public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version"); - public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state"); - public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy"); - public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy"); - public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText"); - public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames"); - public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames"); - public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes"); - public static final String LABELS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "labels"); - public static final String EDGE_PENDING_TASKS_PROPERTY_KEY = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + "__pendingTasks"); + public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames"); + public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames"); + public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version"); + public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state"); + public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY = STATE_PROPERTY_KEY; + public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy"); + public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy"); + public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText"); + public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames"); + public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames"); + public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes"); + public static final String LABELS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "labels"); + public static final String EDGE_PENDING_TASKS_PROPERTY_KEY = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + "__pendingTasks"); /** * Patch vertices property keys. */ @@ -106,7 +104,6 @@ public final class Constants { public static final String PATCH_TYPE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.type"); public static final String PATCH_ACTION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.action"); public static final String PATCH_STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.state"); - /** * The homeId field is used when saving into Atlas a copy of an object that is being imported from another * repository. The homeId will be set to a String that identifies the other repository. The specific format @@ -118,75 +115,60 @@ public final class Constants { * should only be updated via the notifications and calls from Open Metadata Repository Services. */ public static final String HOME_ID_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "homeId"); - /** * The isProxy field is used when saving into Atlas a proxy of an entity - i.e. it is not a whole entity, but * a partial representation of an entity that is referred to by a relationship end. * The isProxy field will be set to true if the entity is a proxy. The field is used during retrieval of an * entity (proxy) from Atlas to indicate that the entity does not contain full entity detail. */ - public static final String IS_PROXY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isProxy"); - + public static final String IS_PROXY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isProxy"); /** * The provenanceType field is used to record the provenance of an instance of an entity or relationship - this * indicates how the instance was created. This corresponds to the InstanceProvenanceType enum defined in ODPi. * To avoid creating a hard dependency on the ODPi class, the value is stored as an int corresponding to the * ordinal in the ODPi enum. */ - public static final String PROVENANCE_TYPE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "provenanceType"); - + public static final String PROVENANCE_TYPE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "provenanceType"); public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp"); - public static final String ENTITY_DELETED_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "entityDeletedTimestamp"); - public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp"); - public static final String IS_INCOMPLETE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isIncomplete"); - /** * search backing index name. */ public static final String BACKING_INDEX = "search"; - /** * search backing index name for vertex keys. */ public static final String VERTEX_INDEX = "vertex_index"; - /** * search backing index name for edge labels. */ public static final String EDGE_INDEX = "edge_index"; - public static final String FULLTEXT_INDEX = "fulltext_index"; - public static final String QUALIFIED_NAME = "Referenceable.qualifiedName"; public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; + public static final String CLASSIFICATION_VERTEX_NAME_KEY = encodePropertyKey(TYPE_NAME_PROPERTY_KEY); public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length"; public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.tags.max-query-str-length"; public static final String INDEX_SEARCH_VERTEX_PREFIX_PROPERTY = "atlas.graph.index.search.vertex.prefix"; public static final String INDEX_SEARCH_VERTEX_PREFIX_DEFAULT = "$v$"; - public static final String MAX_FULLTEXT_QUERY_STR_LENGTH = "atlas.graph.fulltext-max-query-str-length"; public static final String MAX_DSL_QUERY_STR_LENGTH = "atlas.graph.dsl-max-query-str-length"; - public static final String ATTRIBUTE_NAME_GUID = "guid"; public static final String ATTRIBUTE_NAME_TYPENAME = "typeName"; public static final String ATTRIBUTE_NAME_SUPERTYPENAMES = "superTypeNames"; public static final String ATTRIBUTE_NAME_STATE = "state"; public static final String ATTRIBUTE_NAME_VERSION = "version"; public static final String TEMP_STRUCT_NAME_PREFIX = "__tempQueryResultStruct"; - public static final String CLASSIFICATION_ENTITY_GUID = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "entityGuid"); public static final String CLASSIFICATION_ENTITY_STATUS = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "entityStatus"); public static final String CLASSIFICATION_VALIDITY_PERIODS_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "validityPeriods"); public static final String CLASSIFICATION_VERTEX_PROPAGATE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagate"); public static final String CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "removePropagations"); - public static final String CLASSIFICATION_VERTEX_NAME_KEY = encodePropertyKey(TYPE_NAME_PROPERTY_KEY); public static final String CLASSIFICATION_EDGE_NAME_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "name"); public static final String CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isPropagated"); - public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY = STATE_PROPERTY_KEY; public static final String CLASSIFICATION_LABEL = "classifiedAs"; public static final String CLASSIFICATION_NAME_DELIMITER = "|"; public static final String LABEL_NAME_DELIMITER = CLASSIFICATION_NAME_DELIMITER; @@ -205,19 +187,18 @@ public final class Constants { public static final String ATTR_NAME_REFERENCEABLE = "Referenceable."; public static final String ATTR_NAME_REPLICATED_TO = "replicatedTo"; public static final String ATTR_NAME_REPLICATED_FROM = "replicatedFrom"; - public static final Integer INCOMPLETE_ENTITY_VALUE = Integer.valueOf(1); + public static final Integer INCOMPLETE_ENTITY_VALUE = 1; /* * typedef patch constants */ - public static final String TYPEDEF_PATCH_ADD_MANDATORY_ATTRIBUTE = "ADD_MANDATORY_ATTRIBUTE"; + public static final String TYPEDEF_PATCH_ADD_MANDATORY_ATTRIBUTE = "ADD_MANDATORY_ATTRIBUTE"; /* * Task related constants */ public static final String TASK_PREFIX = INTERNAL_PROPERTY_KEY_PREFIX + "task_"; public static final String TASK_TYPE_PROPERTY_KEY = encodePropertyKey(TASK_PREFIX + "v_type"); - public static final String TASK_TYPE_NAME = INTERNAL_PROPERTY_KEY_PREFIX + "AtlasTaskDef"; public static final String TASK_GUID = encodePropertyKey(TASK_PREFIX + "guid"); public static final String TASK_TYPE = encodePropertyKey(TASK_PREFIX + "type"); public static final String TASK_CREATED_TIME = encodePropertyKey(TASK_PREFIX + "timestamp"); @@ -229,7 +210,7 @@ public final class Constants { public static final String TASK_ERROR_MESSAGE = encodePropertyKey(TASK_PREFIX + "errorMessage"); public static final String TASK_START_TIME = encodePropertyKey(TASK_PREFIX + "startTime"); public static final String TASK_END_TIME = encodePropertyKey(TASK_PREFIX + "endTime"); - + public static final String TASK_TYPE_NAME = INTERNAL_PROPERTY_KEY_PREFIX + "AtlasTaskDef"; /** * Index Recovery vertex property keys. */ @@ -248,19 +229,14 @@ public final class Constants { public static final String PROPERTY_KEY_GUIDS_TO_AGEOUT_BY_CUSTOM = encodePropertyKey(AUDIT_REDUCTION_PREFIX + "custom"); public static final String PROPERTY_KEY_GUIDS_TO_SWEEPOUT = encodePropertyKey(AUDIT_REDUCTION_PREFIX + "sweepout"); - public static final String SQOOP_SOURCE = "sqoop"; - public static final String FALCON_SOURCE = "falcon"; - public static final String HBASE_SOURCE = "hbase"; - public static final String HS2_SOURCE = "hive_server2"; - public static final String HMS_SOURCE = "hive_metastore"; - public static final String IMPALA_SOURCE = "impala"; - public static final String STORM_SOURCE = "storm"; - public static final String FILE_SPOOL_SOURCE = "file_spool"; - - /** - * Audit Reduction related constants - */ - public enum AtlasAuditAgingType { DEFAULT, CUSTOM, SWEEP } + public static final String SQOOP_SOURCE = "sqoop"; + public static final String FALCON_SOURCE = "falcon"; + public static final String HBASE_SOURCE = "hbase"; + public static final String HS2_SOURCE = "hive_server2"; + public static final String HMS_SOURCE = "hive_metastore"; + public static final String IMPALA_SOURCE = "impala"; + public static final String STORM_SOURCE = "storm"; + public static final String FILE_SPOOL_SOURCE = "file_spool"; public static final String AUDIT_REDUCTION_TYPE_NAME = "__auditReductionInfo"; public static final String AUDIT_AGING_TYPE_KEY = "auditAgingType"; public static final String AUDIT_AGING_TTL_KEY = "ttl"; @@ -271,11 +247,6 @@ public enum AtlasAuditAgingType { DEFAULT, CUSTOM, SWEEP } public static final String CREATE_EVENTS_AGEOUT_ALLOWED_KEY = "createEventsAgeoutAllowed"; public static final String AUDIT_AGING_SUBTYPES_INCLUDED_KEY = "subTypesIncluded"; - /* - * All supported file-format extensions for Bulk Imports through file upload - */ - public enum SupportedFileExtensions { XLSX, XLS, CSV } - private Constants() { } @@ -283,9 +254,7 @@ private static String getEncodedTypePropertyKey(String defaultKey) { try { Configuration configuration = ApplicationProperties.get(); - if (configuration.containsKey("atlas.graph.index.search.backend") && - configuration.getString("atlas.graph.index.search.backend").equals("elasticsearch")) { - + if (configuration.containsKey("atlas.graph.index.search.backend") && configuration.getString("atlas.graph.index.search.backend").equals("elasticsearch")) { return defaultKey.replaceAll("\\.", "_"); } @@ -294,4 +263,14 @@ private static String getEncodedTypePropertyKey(String defaultKey) { return encodePropertyKey(defaultKey); } } + + /** + * Audit Reduction related constants + */ + public enum AtlasAuditAgingType { DEFAULT, CUSTOM, SWEEP } + + /* + * All supported file-format extensions for Bulk Imports through file upload + */ + public enum SupportedFileExtensions { XLSX, XLS, CSV } } diff --git a/common/src/main/java/org/apache/atlas/service/Service.java b/common/src/main/java/org/apache/atlas/service/Service.java index c87a240faa..6e78659411 100644 --- a/common/src/main/java/org/apache/atlas/service/Service.java +++ b/common/src/main/java/org/apache/atlas/service/Service.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,7 +23,6 @@ * Service interface to start any background jobs. */ public interface Service { - void start() throws AtlasException; void stop() throws AtlasException; diff --git a/common/src/main/java/org/apache/atlas/service/Services.java b/common/src/main/java/org/apache/atlas/service/Services.java index 2d548467fe..01a644c3d8 100644 --- a/common/src/main/java/org/apache/atlas/service/Services.java +++ b/common/src/main/java/org/apache/atlas/service/Services.java @@ -27,6 +27,7 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.inject.Inject; + import java.util.List; import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME; @@ -39,8 +40,9 @@ @Profile("!test") public class Services { public static final Logger LOG = LoggerFactory.getLogger(Services.class); + private static final String DATA_MIGRATION_CLASS_NAME_DEFAULT = "DataMigrationService"; - private static final String FILE_EXTENSION_ZIP = ".zip"; + private static final String FILE_EXTENSION_ZIP = ".zip"; private final List services; private final String dataMigrationClassName; diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java b/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java index ecdf8c5d1c..a60da1ec5d 100644 --- a/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java +++ b/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java @@ -29,11 +29,9 @@ import org.slf4j.LoggerFactory; import java.net.URI; -import java.util.stream.IntStream; +import java.util.Arrays; public class AtlasPathExtractorUtil { - private static final Logger LOG = LoggerFactory.getLogger(AtlasPathExtractorUtil.class); - // Common public static final char QNAME_SEP_METADATA_NAMESPACE = '@'; public static final char QNAME_SEP_ENTITY_NAME = '.'; @@ -41,13 +39,11 @@ public class AtlasPathExtractorUtil { public static final String ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; public static final String ATTRIBUTE_NAME = "name"; public static final String ATTRIBUTE_BUCKET = "bucket"; - // HDFS public static final String HDFS_TYPE_PATH = "hdfs_path"; public static final String ATTRIBUTE_PATH = "path"; public static final String ATTRIBUTE_CLUSTER_NAME = "clusterName"; public static final String ATTRIBUTE_NAMESERVICE_ID = "nameServiceId"; - // AWS S3 public static final String AWS_S3_ATLAS_MODEL_VERSION_V2 = "v2"; public static final String AWS_S3_BUCKET = "aws_s3_bucket"; @@ -60,49 +56,50 @@ public class AtlasPathExtractorUtil { public static final String ATTRIBUTE_OBJECT_PREFIX = "objectPrefix"; public static final String RELATIONSHIP_AWS_S3_BUCKET_S3_PSEUDO_DIRS = "aws_s3_bucket_aws_s3_pseudo_dirs"; public static final String RELATIONSHIP_AWS_S3_V2_CONTAINER_CONTAINED = "aws_s3_v2_container_contained"; - // ADLS Gen2 - public static final String ADLS_GEN2_ACCOUNT = "adls_gen2_account"; - public static final String ADLS_GEN2_CONTAINER = "adls_gen2_container"; - public static final String ADLS_GEN2_DIRECTORY = "adls_gen2_directory"; - public static final String ADLS_GEN2_ACCOUNT_HOST_SUFFIX = ".dfs.core.windows.net"; - public static final String ABFS_SCHEME = "abfs" + SCHEME_SEPARATOR; - public static final String ABFSS_SCHEME = "abfss" + SCHEME_SEPARATOR; - public static final String ATTRIBUTE_ACCOUNT = "account"; - public static final String ATTRIBUTE_PARENT = "parent"; - public static final String RELATIONSHIP_ADLS_GEN2_ACCOUNT_CONTAINERS = "adls_gen2_account_containers"; - public static final String RELATIONSHIP_ADLS_GEN2_PARENT_CHILDREN = "adls_gen2_parent_children"; - + public static final String ADLS_GEN2_ACCOUNT = "adls_gen2_account"; + public static final String ADLS_GEN2_CONTAINER = "adls_gen2_container"; + public static final String ADLS_GEN2_DIRECTORY = "adls_gen2_directory"; + public static final String ADLS_GEN2_ACCOUNT_HOST_SUFFIX = ".dfs.core.windows.net"; + public static final String ABFS_SCHEME = "abfs" + SCHEME_SEPARATOR; + public static final String ABFSS_SCHEME = "abfss" + SCHEME_SEPARATOR; + public static final String ATTRIBUTE_ACCOUNT = "account"; + public static final String ATTRIBUTE_PARENT = "parent"; + public static final String RELATIONSHIP_ADLS_GEN2_ACCOUNT_CONTAINERS = "adls_gen2_account_containers"; + public static final String RELATIONSHIP_ADLS_GEN2_PARENT_CHILDREN = "adls_gen2_parent_children"; // Ozone - public static final String OZONE_VOLUME = "ozone_volume"; - public static final String OZONE_BUCKET = "ozone_bucket"; - public static final String OZONE_KEY = "ozone_key"; - public static final String OZONE_SCHEME = "ofs" + SCHEME_SEPARATOR; - public static final String OZONE_3_SCHEME = "o3fs" + SCHEME_SEPARATOR; - public static final String ATTRIBUTE_VOLUME = "volume"; - public static final String RELATIONSHIP_OZONE_VOLUME_BUCKET = "ozone_volume_buckets"; - public static final String RELATIONSHIP_OZONE_PARENT_CHILDREN = "ozone_parent_children"; - public static final String OZONE_SCHEME_NAME = "ofs"; - + public static final String OZONE_VOLUME = "ozone_volume"; + public static final String OZONE_BUCKET = "ozone_bucket"; + public static final String OZONE_KEY = "ozone_key"; + public static final String OZONE_SCHEME = "ofs" + SCHEME_SEPARATOR; + public static final String OZONE_3_SCHEME = "o3fs" + SCHEME_SEPARATOR; + public static final String ATTRIBUTE_VOLUME = "volume"; + public static final String RELATIONSHIP_OZONE_VOLUME_BUCKET = "ozone_volume_buckets"; + public static final String RELATIONSHIP_OZONE_PARENT_CHILDREN = "ozone_parent_children"; + public static final String OZONE_SCHEME_NAME = "ofs"; //Google Cloud Storage public static final String GCS_SCHEME = "gs" + SCHEME_SEPARATOR; public static final String GCS_BUCKET = "gcp_storage_bucket"; public static final String GCS_VIRTUAL_DIR = "gcp_storage_virtual_directory"; public static final String ATTRIBUTE_GCS_PARENT = "parent"; public static final String RELATIONSHIP_GCS_PARENT_CHILDREN = "gcp_storage_parent_children"; + private static final Logger LOG = LoggerFactory.getLogger(AtlasPathExtractorUtil.class); + + private AtlasPathExtractorUtil() { + // to block instantiation + } public static AtlasEntityWithExtInfo getPathEntity(Path path, PathExtractorContext context) { AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntityWithExtInfo(); - AtlasEntity ret; - String strPath = path.toString(); + String strPath = path.toString(); + AtlasEntity ret; if (context.isConvertPathToLowerCase()) { strPath = strPath.toLowerCase(); } if (isS3Path(strPath)) { - ret = isAwsS3AtlasModelVersionV2(context) ? addS3PathEntityV2(path, entityWithExtInfo, context) : - addS3PathEntityV1(path, entityWithExtInfo, context); + ret = isAwsS3AtlasModelVersionV2(context) ? addS3PathEntityV2(path, entityWithExtInfo, context) : addS3PathEntityV1(path, entityWithExtInfo, context); } else if (isAbfsPath(strPath)) { ret = addAbfsPathEntity(path, entityWithExtInfo, context); } else if (isOzonePath(strPath)) { @@ -119,8 +116,7 @@ public static AtlasEntityWithExtInfo getPathEntity(Path path, PathExtractorConte } private static boolean isAwsS3AtlasModelVersionV2(PathExtractorContext context) { - return StringUtils.isNotEmpty(context.getAwsS3AtlasModelVersion()) && - StringUtils.equalsIgnoreCase(context.getAwsS3AtlasModelVersion(), AWS_S3_ATLAS_MODEL_VERSION_V2); + return StringUtils.isNotEmpty(context.getAwsS3AtlasModelVersion()) && StringUtils.equalsIgnoreCase(context.getAwsS3AtlasModelVersion(), AWS_S3_ATLAS_MODEL_VERSION_V2); } private static boolean isS3Path(String strPath) { @@ -142,9 +138,7 @@ private static boolean isGCSPath(String strPath) { private static AtlasEntity addS3PathEntityV1(Path path, AtlasEntityExtInfo extInfo, PathExtractorContext context) { String strPath = path.toString(); - if (LOG.isDebugEnabled()) { - LOG.debug("==> addS3PathEntityV1(strPath={})", strPath); - } + LOG.debug("==> addS3PathEntityV1(strPath={})", strPath); String metadataNamespace = context.getMetadataNamespace(); String bucketName = path.toUri().getAuthority(); @@ -175,9 +169,7 @@ private static AtlasEntity addS3PathEntityV1(Path path, AtlasEntityExtInfo extIn context.putEntity(pathQualifiedName, ret); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== addS3PathEntityV1(strPath={})", strPath); - } + LOG.debug("<== addS3PathEntityV1(strPath={})", strPath); return ret; } @@ -185,9 +177,7 @@ private static AtlasEntity addS3PathEntityV1(Path path, AtlasEntityExtInfo extIn private static AtlasEntity addS3PathEntityV2(Path path, AtlasEntityExtInfo extInfo, PathExtractorContext context) { String strPath = path.toString(); - if (LOG.isDebugEnabled()) { - LOG.debug("==> addS3PathEntityV2(strPath={})", strPath); - } + LOG.debug("==> addS3PathEntityV2(strPath={})", strPath); String metadataNamespace = context.getMetadataNamespace(); String pathQualifiedName = strPath + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace; @@ -205,9 +195,7 @@ private static AtlasEntity addS3PathEntityV2(Path path, AtlasEntityExtInfo extIn bucketEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, bucketQualifiedName); bucketEntity.setAttribute(ATTRIBUTE_NAME, bucketName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", bucketEntity.getTypeName(), bucketEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", bucketEntity.getTypeName(), bucketEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(bucketQualifiedName, bucketEntity); } @@ -240,9 +228,7 @@ private static AtlasEntity addS3PathEntityV2(Path path, AtlasEntityExtInfo extIn ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, subDirQualifiedName); ret.setAttribute(ATTRIBUTE_NAME, subDirName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(subDirQualifiedName, ret); } @@ -256,9 +242,7 @@ private static AtlasEntity addS3PathEntityV2(Path path, AtlasEntityExtInfo extIn } } - if (LOG.isDebugEnabled()) { - LOG.debug("<== addS3PathEntityV2(strPath={})", strPath); - } + LOG.debug("<== addS3PathEntityV2(strPath={})", strPath); return ret; } @@ -266,9 +250,7 @@ private static AtlasEntity addS3PathEntityV2(Path path, AtlasEntityExtInfo extIn private static AtlasEntity addAbfsPathEntity(Path path, AtlasEntityExtInfo extInfo, PathExtractorContext context) { String strPath = path.toString(); - if (LOG.isDebugEnabled()) { - LOG.debug("==> addAbfsPathEntity(strPath={})", strPath); - } + LOG.debug("==> addAbfsPathEntity(strPath={})", strPath); String metadataNamespace = context.getMetadataNamespace(); String pathQualifiedName = strPath + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace; @@ -288,9 +270,7 @@ private static AtlasEntity addAbfsPathEntity(Path path, AtlasEntityExtInfo extIn storageAcctEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, storageAcctQualifiedName); storageAcctEntity.setAttribute(ATTRIBUTE_NAME, storageAcctName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", storageAcctEntity.getTypeName(), storageAcctEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", storageAcctEntity.getTypeName(), storageAcctEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(storageAcctQualifiedName, storageAcctEntity); } @@ -312,9 +292,7 @@ private static AtlasEntity addAbfsPathEntity(Path path, AtlasEntityExtInfo extIn containerEntity.setAttribute(ATTRIBUTE_NAME, containerName); containerEntity.setRelationshipAttribute(ATTRIBUTE_ACCOUNT, storageAcctObjId); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", containerEntity.getTypeName(), containerEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", containerEntity.getTypeName(), containerEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(containerQualifiedName, containerEntity); } @@ -347,9 +325,7 @@ private static AtlasEntity addAbfsPathEntity(Path path, AtlasEntityExtInfo extIn ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, subDirQualifiedName); ret.setAttribute(ATTRIBUTE_NAME, subDirName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(subDirQualifiedName, ret); } @@ -363,9 +339,7 @@ private static AtlasEntity addAbfsPathEntity(Path path, AtlasEntityExtInfo extIn } } - if (LOG.isDebugEnabled()) { - LOG.debug("<== addAbfsPathEntity(strPath={})", strPath); - } + LOG.debug("<== addAbfsPathEntity(strPath={})", strPath); return ret; } @@ -373,9 +347,7 @@ private static AtlasEntity addAbfsPathEntity(Path path, AtlasEntityExtInfo extIn private static AtlasEntity addOzonePathEntity(Path path, AtlasEntityExtInfo extInfo, PathExtractorContext context) { String strPath = path.toString(); - if (LOG.isDebugEnabled()) { - LOG.debug("==> addOzonePathEntity(strPath={})", strPath); - } + LOG.debug("==> addOzonePathEntity(strPath={})", strPath); String metadataNamespace = context.getMetadataNamespace(); String ozoneScheme = path.toUri().getScheme(); @@ -394,9 +366,7 @@ private static AtlasEntity addOzonePathEntity(Path path, AtlasEntityExtInfo extI volumeEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, volumeQualifiedName); volumeEntity.setAttribute(ATTRIBUTE_NAME, volumeName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", volumeEntity.getTypeName(), volumeEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", volumeEntity.getTypeName(), volumeEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(volumeQualifiedName, volumeEntity); } @@ -413,11 +383,9 @@ private static AtlasEntity addOzonePathEntity(Path path, AtlasEntityExtInfo extI bucketEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, bucketQualifiedName); bucketEntity.setAttribute(ATTRIBUTE_NAME, bucketName); - bucketEntity.setRelationshipAttribute( ATTRIBUTE_VOLUME, AtlasTypeUtil.getAtlasRelatedObjectId(volumeEntity, RELATIONSHIP_OZONE_VOLUME_BUCKET)); + bucketEntity.setRelationshipAttribute(ATTRIBUTE_VOLUME, AtlasTypeUtil.getAtlasRelatedObjectId(volumeEntity, RELATIONSHIP_OZONE_VOLUME_BUCKET)); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", bucketEntity.getTypeName(), bucketEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", bucketEntity.getTypeName(), bucketEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(bucketQualifiedName, bucketEntity); } @@ -433,30 +401,30 @@ private static AtlasEntity addOzonePathEntity(Path path, AtlasEntityExtInfo extI } String keyQNamePrefix = ozoneScheme + SCHEME_SEPARATOR + path.toUri().getAuthority(); - - - String[] subDirNames = dirPath.split(Path.SEPARATOR); + String[] subDirNames = dirPath.split(Path.SEPARATOR); String[] subDirNameArr = subDirNames; + if (ozoneScheme.equals(OZONE_SCHEME_NAME)) { - subDirNames = IntStream.range(3, subDirNameArr.length) - .mapToObj(i -> subDirNameArr[i]) - .toArray(String[]::new); + subDirNames = Arrays.stream(subDirNameArr, 3, subDirNameArr.length).toArray(String[]::new); } boolean volumeBucketAdded = false; + for (String subDirName : subDirNames) { if (StringUtils.isEmpty(subDirName)) { continue; } String subDirPath; + if (ozoneScheme.equals(OZONE_SCHEME_NAME) && !volumeBucketAdded) { - subDirPath = "%s%s" + Path.SEPARATOR + "%s" + Path.SEPARATOR + "%s"; - subDirPath = String.format(subDirPath, parentPath, subDirNameArr[1], subDirNameArr[2], subDirName); + subDirPath = "%s%s" + Path.SEPARATOR + "%s" + Path.SEPARATOR + "%s"; + subDirPath = String.format(subDirPath, parentPath, subDirNameArr[1], subDirNameArr[2], subDirName); volumeBucketAdded = true; } else { subDirPath = parentPath + subDirName; } + String subDirQualifiedName = keyQNamePrefix + subDirPath + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace; ret = context.getEntity(subDirQualifiedName); @@ -468,21 +436,17 @@ private static AtlasEntity addOzonePathEntity(Path path, AtlasEntityExtInfo extI ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, subDirQualifiedName); ret.setAttribute(ATTRIBUTE_NAME, subDirName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(subDirQualifiedName, ret); } parentObjId = AtlasTypeUtil.getAtlasRelatedObjectId(ret, RELATIONSHIP_OZONE_PARENT_CHILDREN); - parentPath = subDirPath + Path.SEPARATOR;; + parentPath = subDirPath + Path.SEPARATOR; } } - if (LOG.isDebugEnabled()) { - LOG.debug("<== addOzonePathEntity(strPath={})", strPath); - } + LOG.debug("<== addOzonePathEntity(strPath={})", strPath); return ret; } @@ -490,9 +454,7 @@ private static AtlasEntity addOzonePathEntity(Path path, AtlasEntityExtInfo extI private static AtlasEntity addGCSPathEntity(Path path, AtlasEntityExtInfo extInfo, PathExtractorContext context) { String strPath = path.toString(); - if (LOG.isDebugEnabled()) { - LOG.debug("==> addGCSPathEntity(strPath={})", strPath); - } + LOG.debug("==> addGCSPathEntity(strPath={})", strPath); String metadataNamespace = context.getMetadataNamespace(); String pathQualifiedName = strPath + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace; @@ -510,9 +472,7 @@ private static AtlasEntity addGCSPathEntity(Path path, AtlasEntityExtInfo extInf bucketEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, bucketQualifiedName); bucketEntity.setAttribute(ATTRIBUTE_NAME, bucketName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", bucketEntity.getTypeName(), bucketEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", bucketEntity.getTypeName(), bucketEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(bucketQualifiedName, bucketEntity); } @@ -545,9 +505,7 @@ private static AtlasEntity addGCSPathEntity(Path path, AtlasEntityExtInfo extInf ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, subDirQualifiedName); ret.setAttribute(ATTRIBUTE_NAME, subDirName); - if (LOG.isDebugEnabled()) { - LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - } + LOG.debug("adding entity: typeName={}, qualifiedName={}", ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); context.putEntity(subDirQualifiedName, ret); } @@ -561,9 +519,7 @@ private static AtlasEntity addGCSPathEntity(Path path, AtlasEntityExtInfo extInf } } - if (LOG.isDebugEnabled()) { - LOG.debug("<== addGCSPathEntity(strPath={})", strPath); - } + LOG.debug("<== addGCSPathEntity(strPath={})", strPath); return ret; } @@ -575,9 +531,7 @@ private static AtlasEntity addHDFSPathEntity(Path path, PathExtractorContext con strPath = strPath.toLowerCase(); } - if (LOG.isDebugEnabled()) { - LOG.debug("==> addHDFSPathEntity(strPath={})", strPath); - } + LOG.debug("==> addHDFSPathEntity(strPath={})", strPath); String nameServiceID = HdfsNameServiceResolver.getNameServiceIDForPath(strPath); String attrPath = StringUtils.isEmpty(nameServiceID) ? strPath : HdfsNameServiceResolver.getPathWithNameServiceID(strPath); @@ -605,9 +559,7 @@ private static AtlasEntity addHDFSPathEntity(Path path, PathExtractorContext con context.putEntity(pathQualifiedName, ret); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== addHDFSPathEntity(strPath={})", strPath); - } + LOG.debug("<== addHDFSPathEntity(strPath={})", strPath); return ret; } @@ -625,32 +577,39 @@ private static String getAbfsStorageAccountName(URI uri) { } private static String getOzoneVolumeName(Path path) { - String strPath = path.toString(); + String strPath = path.toString(); String volumeName = StringUtils.EMPTY; + if (strPath.startsWith(OZONE_3_SCHEME)) { String pathAuthority = path.toUri().getAuthority(); volumeName = pathAuthority.split("\\.")[1]; } else if (strPath.startsWith(OZONE_SCHEME)) { strPath = strPath.replaceAll(OZONE_SCHEME, StringUtils.EMPTY); + if (strPath.split(Path.SEPARATOR).length >= 2) { volumeName = strPath.split(Path.SEPARATOR)[1]; } } + return volumeName; } private static String getOzoneBucketName(Path path) { - String strPath = path.toString(); + String strPath = path.toString(); String bucketName = StringUtils.EMPTY; + if (strPath.startsWith(OZONE_3_SCHEME)) { String pathAuthority = path.toUri().getAuthority(); + bucketName = pathAuthority.split("\\.")[0]; } else if (strPath.startsWith(OZONE_SCHEME)) { strPath = strPath.replaceAll(OZONE_SCHEME, StringUtils.EMPTY); + if (strPath.split(Path.SEPARATOR).length >= 3) { bucketName = strPath.split(Path.SEPARATOR)[2]; } } + return bucketName; } diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java index a65dc86063..11a25b5cf1 100644 --- a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java +++ b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java @@ -18,15 +18,13 @@ package org.apache.atlas.utils; - import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; public class AtlasPerfMetrics { - private final Map metrics = new LinkedHashMap<>(); - private long startTimeMs = -1; - + private final Map metrics = new LinkedHashMap<>(); + private long startTimeMs = -1; public MetricRecorder getMetricRecorder(String name) { return new MetricRecorder(name); @@ -41,13 +39,7 @@ public void recordMetric(MetricRecorder recorder) { startTimeMs = System.currentTimeMillis(); } - Metric metric = metrics.get(name); - - if (metric == null) { - metric = new Metric(name); - - metrics.put(name, metric); - } + Metric metric = metrics.computeIfAbsent(name, Metric::new); metric.invocations++; metric.totalTimeMSecs += timeTaken; @@ -91,23 +83,10 @@ public String toString() { return sb.toString(); } - public class MetricRecorder { - private final String name; - private final long startTimeMs = System.currentTimeMillis(); - - MetricRecorder(String name) { - this.name = name; - } - - long getElapsedTime() { - return System.currentTimeMillis() - startTimeMs; - } - } - public static class Metric { private final String name; - private short invocations = 0; - private long totalTimeMSecs = 0; + private short invocations; + private long totalTimeMSecs; public Metric(String name) { this.name = name; @@ -125,4 +104,17 @@ public long getTotalTimeMSecs() { return totalTimeMSecs; } } + + public static class MetricRecorder { + private final String name; + private final long startTimeMs = System.currentTimeMillis(); + + MetricRecorder(String name) { + this.name = name; + } + + long getElapsedTime() { + return System.currentTimeMillis() - startTimeMs; + } + } } diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfTracer.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfTracer.java index a2baa65b03..1a4b92fda8 100644 --- a/common/src/main/java/org/apache/atlas/utils/AtlasPerfTracer.java +++ b/common/src/main/java/org/apache/atlas/utils/AtlasPerfTracer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,16 +20,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Handles logging of performance measurements. */ public final class AtlasPerfTracer { - protected final Logger logger; - protected final String tag; - private final long startTimeMs; + private static final long reportingThresholdMs = 0L; + + private final Logger logger; + private final String tag; + private final long startTimeMs; - private static long reportingThresholdMs = 0L; + private AtlasPerfTracer(Logger logger, String tag) { + this.logger = logger; + this.tag = tag; + startTimeMs = System.currentTimeMillis(); + } public static Logger getPerfLogger(String name) { return LoggerFactory.getLogger("org.apache.atlas.perf." + name); @@ -53,12 +58,6 @@ public static void log(AtlasPerfTracer tracer) { } } - private AtlasPerfTracer(Logger logger, String tag) { - this.logger = logger; - this.tag = tag; - startTimeMs = System.currentTimeMillis(); - } - public String getTag() { return tag; } @@ -73,6 +72,7 @@ public long getElapsedTime() { public void log() { long elapsedTime = getElapsedTime(); + if (elapsedTime > reportingThresholdMs) { logger.debug("PERF|{}|{}", tag, elapsedTime); } diff --git a/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java b/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java index 4b9d0bef22..fc4c622e81 100644 --- a/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java +++ b/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java @@ -28,11 +28,10 @@ import java.util.Map; import java.util.Objects; - public class HdfsNameServiceResolver { private static final Logger LOG = LoggerFactory.getLogger(HdfsNameServiceResolver.class); - public static final String HDFS_SCHEME = "hdfs://"; + public static final String HDFS_SCHEME = "hdfs://"; private static final int DEFAULT_PORT = 8020; private static final String HDFS_NAMESERVICE_PROPERTY_KEY = "dfs.nameservices"; @@ -43,20 +42,18 @@ public class HdfsNameServiceResolver { private static final Map hostToNameServiceMap = new HashMap<>(); - static { - init(); + private HdfsNameServiceResolver() { + // to block instantiation } public static String getPathWithNameServiceID(String path) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> HdfsNameServiceResolver.getPathWithNameServiceID({})", path); - } + LOG.debug("==> HdfsNameServiceResolver.getPathWithNameServiceID({})", path); String ret = path; // Only handle URLs that begin with hdfs:// if (path != null && path.indexOf(HDFS_SCHEME) == 0) { - URI uri = new Path(path).toUri(); + URI uri = new Path(path).toUri(); String nsId; if (uri.getPort() != -1) { @@ -70,18 +67,13 @@ public static String getPathWithNameServiceID(String path) { } } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== HdfsNameServiceResolver.getPathWithNameServiceID({}) = {}", path, ret); - } + LOG.debug("<== HdfsNameServiceResolver.getPathWithNameServiceID({}) = {}", path, ret); return ret; } public static String getNameServiceIDForPath(String path) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> HdfsNameServiceResolver.getNameServiceIDForPath({})", path); - } + LOG.debug("==> HdfsNameServiceResolver.getNameServiceIDForPath({})", path); String ret = ""; @@ -110,64 +102,53 @@ public static String getNameServiceIDForPath(String path) { } } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== HdfsNameServiceResolver.getNameServiceIDForPath({}) = {}", path, ret); - } + LOG.debug("<== HdfsNameServiceResolver.getNameServiceIDForPath({}) = {}", path, ret); return ret; } private static String getNameServiceID(String host, int port) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> HdfsNameServiceResolver.getNameServiceID({}, {})", host, port); - } + LOG.debug("==> HdfsNameServiceResolver.getNameServiceID({}, {})", host, port); String ret = hostToNameServiceMap.getOrDefault(host + ":" + port, ""); - if (LOG.isDebugEnabled()) { - LOG.debug("<== HdfsNameServiceResolver.getNameServiceID({}, {}) = {}", host, port, ret); - } + LOG.debug("<== HdfsNameServiceResolver.getNameServiceID({}, {}) = {}", host, port, ret); return ret; } private static void init() { - if (LOG.isDebugEnabled()) { - LOG.debug("==> HdfsNameServiceResolver.init()"); - } + LOG.debug("==> HdfsNameServiceResolver.init()"); + HdfsConfiguration hdfsConfiguration = new HdfsConfiguration(true); // Determine all available nameServiceIDs String[] nameServiceIDs = hdfsConfiguration.getTrimmedStrings(HDFS_INTERNAL_NAMESERVICE_PROPERTY_KEY); + if (Objects.isNull(nameServiceIDs) || nameServiceIDs.length == 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("NSID not found for {}, looking under {}", HDFS_INTERNAL_NAMESERVICE_PROPERTY_KEY, HDFS_NAMESERVICE_PROPERTY_KEY); - } + LOG.debug("NSID not found for {}, looking under {}", HDFS_INTERNAL_NAMESERVICE_PROPERTY_KEY, HDFS_NAMESERVICE_PROPERTY_KEY); + // Attempt another lookup using internal name service IDs key nameServiceIDs = hdfsConfiguration.getTrimmedStrings(HDFS_NAMESERVICE_PROPERTY_KEY); } if (Objects.nonNull(nameServiceIDs) && nameServiceIDs.length > 0) { - if (LOG.isDebugEnabled()) { - LOG.debug("NSIDs = {}", nameServiceIDs); - } + LOG.debug("NSIDs = {}", (Object) nameServiceIDs); boolean isHA; + for (String nameServiceID : nameServiceIDs) { // Find NameNode addresses and map to the NameServiceID String[] nameNodes = hdfsConfiguration.getTrimmedStrings(HDFS_NAMENODES_HA_NODES_PREFIX + nameServiceID); + isHA = nameNodes != null && nameNodes.length > 0; - String nameNodeMappingKey, nameNodeAddress; if (isHA) { - if (LOG.isDebugEnabled()) { - LOG.debug("Processing HA node info"); - } + LOG.debug("Processing HA node info"); for (String nameNode : nameNodes) { - nameNodeMappingKey = String.format(HDFS_NAMENODE_HA_ADDRESS_TEMPLATE, nameServiceID, nameNode); - nameNodeAddress = hdfsConfiguration.get(nameNodeMappingKey, ""); + String nameNodeMappingKey = String.format(HDFS_NAMENODE_HA_ADDRESS_TEMPLATE, nameServiceID, nameNode); + String nameNodeAddress = hdfsConfiguration.get(nameNodeMappingKey, ""); // Add a mapping only if found if (StringUtils.isNotEmpty(nameNodeAddress)) { @@ -175,8 +156,8 @@ private static void init() { } } } else { - nameNodeMappingKey = String.format(HDFS_NAMENODE_ADDRESS_TEMPLATE, nameServiceID); - nameNodeAddress = hdfsConfiguration.get(nameNodeMappingKey, ""); + String nameNodeMappingKey = String.format(HDFS_NAMENODE_ADDRESS_TEMPLATE, nameServiceID); + String nameNodeAddress = hdfsConfiguration.get(nameNodeMappingKey, ""); // Add a mapping only if found if (StringUtils.isNotEmpty(nameNodeAddress)) { @@ -185,13 +166,13 @@ private static void init() { } } } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No NSID could be resolved"); - } + LOG.debug("No NSID could be resolved"); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== HdfsNameServiceResolver.init()"); - } + LOG.debug("<== HdfsNameServiceResolver.init()"); + } + + static { + init(); } } diff --git a/common/src/main/java/org/apache/atlas/utils/KafkaUtils.java b/common/src/main/java/org/apache/atlas/utils/KafkaUtils.java index 672caa8e60..97f738e161 100644 --- a/common/src/main/java/org/apache/atlas/utils/KafkaUtils.java +++ b/common/src/main/java/org/apache/atlas/utils/KafkaUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,10 +26,10 @@ import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.ListTopicsOptions; import org.apache.kafka.clients.admin.ListTopicsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.admin.ListTopicsOptions; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.errors.TopicExistsException; @@ -44,12 +44,16 @@ import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; + import static org.apache.atlas.security.SecurityProperties.HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH; public class KafkaUtils implements AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(KafkaUtils.class); + public static final String ATLAS_KAFKA_PROPERTY_PREFIX = "atlas.kafka"; + public static final String KAFKA_SASL_JAAS_CONFIG_PROPERTY = "sasl.jaas.config"; + public static final String JAAS_PASSWORD_SUFFIX = "password"; + private static final String JAAS_CONFIG_PREFIX_PARAM = "atlas.jaas"; private static final String JAAS_CONFIG_LOGIN_MODULE_NAME_PARAM = "loginModuleName"; private static final String JAAS_CONFIG_LOGIN_MODULE_CONTROL_FLAG_PARAM = "loginModuleControlFlag"; @@ -60,21 +64,14 @@ public class KafkaUtils implements AutoCloseable { private static final String JAAS_DEFAULT_CLIENT_NAME = "KafkaClient"; private static final String JAAS_TICKET_BASED_CLIENT_NAME = "ticketBased-KafkaClient"; private static final String IMPORT_INTERNAL_TOPICS = "atlas.kafka.bridge.enable.internal.topics.import"; + private static final String JAAS_MASK_PASSWORD = "********"; - public static final String ATLAS_KAFKA_PROPERTY_PREFIX = "atlas.kafka"; - public static final String KAFKA_SASL_JAAS_CONFIG_PROPERTY = "sasl.jaas.config"; - - public static final String JAAS_PASSWORD_SUFFIX = "password"; - private static final String JAAS_MASK_PASSWORD = "********"; - - final protected Properties kafkaConfiguration; - final protected AdminClient adminClient; - final protected boolean importInternalTopics; + protected final Properties kafkaConfiguration; + protected final AdminClient adminClient; + protected final boolean importInternalTopics; public KafkaUtils(Configuration atlasConfiguration) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> KafkaUtils() "); - } + LOG.debug("==> KafkaUtils() "); this.kafkaConfiguration = ApplicationProperties.getSubsetAsProperties(atlasConfiguration, ATLAS_KAFKA_PROPERTY_PREFIX); @@ -83,124 +80,11 @@ public KafkaUtils(Configuration atlasConfiguration) { adminClient = AdminClient.create(this.kafkaConfiguration); importInternalTopics = atlasConfiguration.getBoolean(IMPORT_INTERNAL_TOPICS, false); - if (LOG.isDebugEnabled()) { - LOG.debug("<== KafkaUtils() "); - } - } - - public void createTopics(List topicNames, int numPartitions, int replicationFactor) - throws TopicExistsException, ExecutionException, InterruptedException { - - if (LOG.isDebugEnabled()) { - LOG.debug("==> createTopics() "); - } - - List newTopicList = topicNames.stream() - .map(topicName -> new NewTopic(topicName, numPartitions, (short) replicationFactor)) - .collect(Collectors.toList()); - - CreateTopicsResult createTopicsResult = adminClient.createTopics(newTopicList); - Map> futureMap = createTopicsResult.values(); - - for (Map.Entry> futureEntry : futureMap.entrySet()) { - KafkaFuture future = futureEntry.getValue(); - - future.get(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== createTopics() "); - } - } - - public List listAllTopics() throws ExecutionException, InterruptedException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> KafkaUtils.listAllTopics() "); - } - - ListTopicsResult listTopicsResult = adminClient.listTopics((new ListTopicsOptions()).listInternal(importInternalTopics)); - List ret = new ArrayList<>(listTopicsResult.names().get()); - - if (LOG.isDebugEnabled()) { - LOG.debug("<== KafkaUtils.listAllTopics() "); - } - - return ret; - } - - public Integer getPartitionCount(String topicName) throws ExecutionException, InterruptedException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> KafkaUtils.getPartitionCount({})", topicName); - } - - Integer ret = null; - List partitionList = getPartitionList(topicName); - - if (partitionList != null) { - ret = partitionList.size(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== KafkaUtils.getPartitionCount returning for topic {} with count {}", topicName, ret); - } - - return ret; - } - - public Integer getReplicationFactor(String topicName) throws ExecutionException, InterruptedException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> KafkaUtils.getReplicationFactor({})", topicName); - } - - Integer ret = null; - List partitionList = getPartitionList(topicName); - - if (partitionList != null) { - ret = partitionList.stream().mapToInt(x -> x.replicas().size()).max().getAsInt(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== KafkaUtils.getReplicationFactor returning for topic {} with replicationFactor {}", topicName, ret); - } - - return ret; - } - - private List getPartitionList(String topicName) throws ExecutionException, InterruptedException { - List ret = null; - DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singleton(topicName)); - - if (describeTopicsResult != null) { - Map> futureMap = describeTopicsResult.values(); - - for (Map.Entry> futureEntry : futureMap.entrySet()) { - KafkaFuture topicDescriptionFuture = futureEntry.getValue(); - TopicDescription topicDescription = topicDescriptionFuture.get(); - - ret = topicDescription.partitions(); - } - } - - return ret; - } - - public void close() { - if (LOG.isDebugEnabled()) { - LOG.debug("==> KafkaUtils.close()"); - } - if (adminClient != null) { - adminClient.close(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== KafkaUtils.close()"); - } + LOG.debug("<== KafkaUtils() "); } public static void setKafkaJAASProperties(Configuration configuration, Properties kafkaProperties) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> KafkaUtils.setKafkaJAASProperties()"); - } + LOG.debug("==> KafkaUtils.setKafkaJAASProperties()"); if (kafkaProperties.containsKey(KAFKA_SASL_JAAS_CONFIG_PROPERTY)) { LOG.debug("JAAS config is already set, returning"); @@ -216,18 +100,14 @@ public static void setKafkaJAASProperties(Configuration configuration, Propertie // Required for backward compatability for Hive CLI if (!isLoginKeytabBased() && isLoginTicketBased()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Checking if ticketBased-KafkaClient is set"); - } + LOG.debug("Checking if ticketBased-KafkaClient is set"); // if ticketBased-KafkaClient property is not specified then use the default client name String ticketBasedConfigPrefix = JAAS_CONFIG_PREFIX_PARAM + "." + JAAS_TICKET_BASED_CLIENT_NAME; Configuration ticketBasedConfig = configuration.subset(ticketBasedConfigPrefix); if (ticketBasedConfig != null && !ticketBasedConfig.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("ticketBased-KafkaClient JAAS configuration is set, using it"); - } + LOG.debug("ticketBased-KafkaClient JAAS configuration is set, using it"); jaasClientName = JAAS_TICKET_BASED_CLIENT_NAME; } else { @@ -257,11 +137,11 @@ public static void setKafkaJAASProperties(Configuration configuration, Propertie LOG.warn("Unknown JAAS configuration value for ({}) = [{}], valid value are [{}] using the default value, REQUIRED", keyParam, controlFlag, validValues); } - String optionPrefix = keyPrefix + JAAS_CONFIG_LOGIN_OPTIONS_PREFIX + "."; - String principalOptionKey = optionPrefix + JAAS_PRINCIPAL_PROP; - String passwordOptionKey = optionPrefix + JAAS_PASSWORD_SUFFIX; - int optionPrefixLen = optionPrefix.length(); - StringBuffer optionStringBuffer = new StringBuffer(); + String optionPrefix = keyPrefix + JAAS_CONFIG_LOGIN_OPTIONS_PREFIX + "."; + String principalOptionKey = optionPrefix + JAAS_PRINCIPAL_PROP; + String passwordOptionKey = optionPrefix + JAAS_PASSWORD_SUFFIX; + int optionPrefixLen = optionPrefix.length(); + StringBuilder optionStringBuffer = new StringBuilder(); for (String key : jaasConfig.stringPropertyNames()) { if (key.startsWith(optionPrefix)) { @@ -277,8 +157,10 @@ public static void setKafkaJAASProperties(Configuration configuration, Propertie } catch (IOException e) { LOG.warn("Failed to build serverPrincipal. Using provided value:[{}]", optionVal); } + if (key.equalsIgnoreCase(passwordOptionKey)) { String jaasKafkaClientConfigurationProperty = "atlas.jaas.KafkaClient.option.password"; + if (JAAS_MASK_PASSWORD.equals(configuration.getString(jaasKafkaClientConfigurationProperty))) { try { optionVal = SecurityUtil.getPassword(configuration, jaasKafkaClientConfigurationProperty, HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH); @@ -287,6 +169,7 @@ public static void setKafkaJAASProperties(Configuration configuration, Propertie } } } + optionVal = surroundWithQuotes(optionVal); optionStringBuffer.append(String.format(" %s=%s", key.substring(optionPrefixLen), optionVal)); @@ -294,35 +177,33 @@ public static void setKafkaJAASProperties(Configuration configuration, Propertie } } - String newJaasProperty = String.format("%s %s %s ;", loginModuleName.trim(), controlFlag, optionStringBuffer.toString()); + String newJaasProperty = String.format("%s %s %s ;", loginModuleName.trim(), controlFlag, optionStringBuffer); kafkaProperties.put(KAFKA_SASL_JAAS_CONFIG_PROPERTY, newJaasProperty); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== KafkaUtils.setKafkaJAASProperties()"); - } + LOG.debug("<== KafkaUtils.setKafkaJAASProperties()"); } - static boolean isLoginKeytabBased() { + public static boolean isLoginTicketBased() { boolean ret = false; try { - ret = UserGroupInformation.isLoginKeytabBased(); + ret = UserGroupInformation.isLoginTicketBased(); } catch (Exception excp) { - LOG.warn("Error in determining keytab for KafkaClient-JAAS config", excp); + LOG.warn("Error in determining ticket-cache for KafkaClient-JAAS config", excp); } return ret; } - public static boolean isLoginTicketBased() { + static boolean isLoginKeytabBased() { boolean ret = false; try { - ret = UserGroupInformation.isLoginTicketBased(); + ret = UserGroupInformation.isLoginKeytabBased(); } catch (Exception excp) { - LOG.warn("Error in determining ticket-cache for KafkaClient-JAAS config", excp); + LOG.warn("Error in determining keytab for KafkaClient-JAAS config", excp); } return ret; @@ -338,4 +219,92 @@ static String surroundWithQuotes(String optionVal) { String doubleQuoteEscaped = optionVal.replace("\"", "\\\""); return String.format("\"%s\"", doubleQuoteEscaped); } -} \ No newline at end of file + + public void createTopics(List topicNames, int numPartitions, int replicationFactor) throws TopicExistsException, ExecutionException, InterruptedException { + LOG.debug("==> createTopics() "); + + List newTopicList = topicNames.stream() + .map(topicName -> new NewTopic(topicName, numPartitions, (short) replicationFactor)) + .collect(Collectors.toList()); + + CreateTopicsResult createTopicsResult = adminClient.createTopics(newTopicList); + Map> futureMap = createTopicsResult.values(); + + for (Map.Entry> futureEntry : futureMap.entrySet()) { + KafkaFuture future = futureEntry.getValue(); + + future.get(); + } + + LOG.debug("<== createTopics() "); + } + + public List listAllTopics() throws ExecutionException, InterruptedException { + LOG.debug("==> KafkaUtils.listAllTopics() "); + + ListTopicsResult listTopicsResult = adminClient.listTopics((new ListTopicsOptions()).listInternal(importInternalTopics)); + List ret = new ArrayList<>(listTopicsResult.names().get()); + + LOG.debug("<== KafkaUtils.listAllTopics() "); + + return ret; + } + + public Integer getPartitionCount(String topicName) throws ExecutionException, InterruptedException { + LOG.debug("==> KafkaUtils.getPartitionCount({})", topicName); + + Integer ret = null; + List partitionList = getPartitionList(topicName); + + if (partitionList != null) { + ret = partitionList.size(); + } + + LOG.debug("<== KafkaUtils.getPartitionCount returning for topic {} with count {}", topicName, ret); + + return ret; + } + + public Integer getReplicationFactor(String topicName) throws ExecutionException, InterruptedException { + LOG.debug("==> KafkaUtils.getReplicationFactor({})", topicName); + + Integer ret = null; + List partitionList = getPartitionList(topicName); + + if (partitionList != null) { + ret = partitionList.stream().mapToInt(x -> x.replicas().size()).max().orElse(1); + } + + LOG.debug("<== KafkaUtils.getReplicationFactor returning for topic {} with replicationFactor {}", topicName, ret); + + return ret; + } + + public void close() { + LOG.debug("==> KafkaUtils.close()"); + + if (adminClient != null) { + adminClient.close(); + } + + LOG.debug("<== KafkaUtils.close()"); + } + + private List getPartitionList(String topicName) throws ExecutionException, InterruptedException { + List ret = null; + DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singleton(topicName)); + + if (describeTopicsResult != null) { + Map> futureMap = describeTopicsResult.values(); + + for (Map.Entry> futureEntry : futureMap.entrySet()) { + KafkaFuture topicDescriptionFuture = futureEntry.getValue(); + TopicDescription topicDescription = topicDescriptionFuture.get(); + + ret = topicDescription.partitions(); + } + } + + return ret; + } +} diff --git a/common/src/main/java/org/apache/atlas/utils/LruCache.java b/common/src/main/java/org/apache/atlas/utils/LruCache.java index dcaffef6d5..01185703fc 100644 --- a/common/src/main/java/org/apache/atlas/utils/LruCache.java +++ b/common/src/main/java/org/apache/atlas/utils/LruCache.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,39 +17,37 @@ */ package org.apache.atlas.utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.text.DateFormat; import java.util.Date; import java.util.LinkedHashMap; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * Fixed size LRU Cache. * */ -public class LruCache extends LinkedHashMap{ +public class LruCache extends LinkedHashMap { + private static final Logger LOGGER = LoggerFactory.getLogger(LruCache.class.getName()); private static final long serialVersionUID = 8715233786643882558L; - private static final Logger LOGGER = LoggerFactory.getLogger(LruCache.class.getName()); - /** * Specifies the number evictions that pass before a warning is logged. */ private final int evictionWarningThrottle; + // The maximum number of entries the cache holds. + private final int capacity; + // The number of evictions since the last warning was logged. - private long evictionsSinceWarning = 0; + private long evictionsSinceWarning; // When the last eviction warning was issued. private Date lastEvictionWarning = new Date(); - // The maximum number of entries the cache holds. - private final int capacity; - - /** * * @param cacheSize The size of the cache. @@ -57,16 +55,19 @@ public class LruCache extends LinkedHashMap{ */ public LruCache(int cacheSize, int evictionWarningThrottle) { super(cacheSize, 0.75f, true); + this.evictionWarningThrottle = evictionWarningThrottle; - this.capacity = cacheSize; + this.capacity = cacheSize; } @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - if( size() > capacity) { + protected boolean removeEldestEntry(Map.Entry eldest) { + if (size() > capacity) { evictionWarningIfNeeded(); + return true; } + return false; } @@ -84,14 +85,13 @@ private void evictionWarningIfNeeded() { if (evictionsSinceWarning >= evictionWarningThrottle) { DateFormat dateFormat = DateFormat.getDateTimeInstance(); + if (LOGGER.isInfoEnabled()) { - LOGGER.info("There have been " + evictionsSinceWarning - + " evictions from the cache since " - + dateFormat.format(lastEvictionWarning)); + LOGGER.info("There have been {} evictions from the cache since {}", evictionsSinceWarning, dateFormat.format(lastEvictionWarning)); } + evictionsSinceWarning = 0; - lastEvictionWarning = new Date(); + lastEvictionWarning = new Date(); } } - } diff --git a/common/src/main/java/org/apache/atlas/utils/OnAtlasPropertyCondition.java b/common/src/main/java/org/apache/atlas/utils/OnAtlasPropertyCondition.java index 7259036be6..f70157c6a2 100644 --- a/common/src/main/java/org/apache/atlas/utils/OnAtlasPropertyCondition.java +++ b/common/src/main/java/org/apache/atlas/utils/OnAtlasPropertyCondition.java @@ -30,26 +30,31 @@ import org.springframework.core.type.AnnotationMetadata; public class OnAtlasPropertyCondition implements Condition { - private final Logger LOG = LoggerFactory.getLogger(OnAtlasPropertyCondition.class); + private static final Logger LOG = LoggerFactory.getLogger(OnAtlasPropertyCondition.class); @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - boolean matches = false; - String propertyName = (String) metadata.getAnnotationAttributes(ConditionalOnAtlasProperty.class.getName()).get("property"); - boolean isDefault = (Boolean) metadata.getAnnotationAttributes(ConditionalOnAtlasProperty.class.getName()).get("isDefault"); - if (metadata instanceof AnnotatedTypeMetadata) { + boolean matches = false; + String propertyName = (String) metadata.getAnnotationAttributes(ConditionalOnAtlasProperty.class.getName()).get("property"); + boolean isDefault = (Boolean) metadata.getAnnotationAttributes(ConditionalOnAtlasProperty.class.getName()).get("isDefault"); + + if (metadata instanceof AnnotationMetadata) { String className = ((AnnotationMetadata) metadata).getClassName(); try { - Configuration configuration = ApplicationProperties.get(); - String configuredProperty = configuration.getString(propertyName); + Configuration configuration = ApplicationProperties.get(); + String configuredProperty = configuration.getString(propertyName); + if (StringUtils.isNotEmpty(configuredProperty)) { matches = configuredProperty.equals(className); - } else if (isDefault) matches = true; + } else if (isDefault) { + matches = true; + } } catch (AtlasException e) { LOG.error("Unable to load atlas properties. Dependent bean configuration may fail"); } } + return matches; } } diff --git a/common/src/main/java/org/apache/atlas/utils/ParamChecker.java b/common/src/main/java/org/apache/atlas/utils/ParamChecker.java index c8d7464dcd..e0a50dfb1b 100644 --- a/common/src/main/java/org/apache/atlas/utils/ParamChecker.java +++ b/common/src/main/java/org/apache/atlas/utils/ParamChecker.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,6 @@ * Utilities for checking parameters. */ public final class ParamChecker { - private ParamChecker() { } @@ -39,6 +38,7 @@ public static T notNull(T obj, String name) { if (obj == null) { throw new IllegalArgumentException(name + " cannot be null"); } + return obj; } @@ -50,9 +50,11 @@ public static T notNull(T obj, String name) { */ public static Collection notNullElements(Collection list, String name) { notEmpty(list, name); + for (T ele : list) { notNull(ele, String.format("Collection %s element %s", name, ele)); } + return list; } @@ -64,9 +66,11 @@ public static Collection notNullElements(Collection list, String name) */ public static T[] notNullElements(T[] array, String name) { notEmpty(Arrays.asList(array), name); + for (T ele : array) { notNull(ele, String.format("Collection %s element %s", name, ele)); } + return array; } @@ -77,9 +81,11 @@ public static T[] notNullElements(T[] array, String name) { */ public static Collection notEmpty(Collection list, String name) { notNull(list, name); + if (list.isEmpty()) { throw new IllegalArgumentException(String.format("Collection %s is empty", name)); } + return list; } @@ -117,9 +123,10 @@ public static String notEmptyIfNotNull(String value, String name, String info) { return value; } - if (value.trim().length() == 0) { + if (value.trim().isEmpty()) { throw new IllegalArgumentException(name + " cannot be empty" + (info == null ? "" : ", " + info)); } + return value.trim(); } @@ -135,6 +142,7 @@ public static String notEmpty(String value, String name, String info) { if (value == null) { throw new IllegalArgumentException(name + " cannot be null" + (info == null ? "" : ", " + info)); } + return notEmptyIfNotNull(value, name, info); } @@ -148,6 +156,7 @@ public static void lessThan(long value, long maxValue, String name) { if (value <= 0) { throw new IllegalArgumentException(name + " should be > 0, current value " + value); } + if (value > maxValue) { throw new IllegalArgumentException(name + " should be <= " + maxValue + ", current value " + value); } diff --git a/common/src/main/java/org/apache/atlas/utils/PathExtractorContext.java b/common/src/main/java/org/apache/atlas/utils/PathExtractorContext.java index ce688e4e42..b6bb3622dd 100644 --- a/common/src/main/java/org/apache/atlas/utils/PathExtractorContext.java +++ b/common/src/main/java/org/apache/atlas/utils/PathExtractorContext.java @@ -30,15 +30,15 @@ public class PathExtractorContext { private final String awsS3AtlasModelVersion; public PathExtractorContext(String metadataNamespace) { - this(metadataNamespace, new HashMap<>(), false, null) ; + this(metadataNamespace, new HashMap<>(), false, null); } public PathExtractorContext(String metadataNamespace, String awsS3AtlasModelVersion) { - this(metadataNamespace, new HashMap<>(), false, awsS3AtlasModelVersion) ; + this(metadataNamespace, new HashMap<>(), false, awsS3AtlasModelVersion); } public PathExtractorContext(String metadataNamespace, boolean isConvertPathToLowerCase, String awsS3AtlasModelVersion) { - this(metadataNamespace, new HashMap<>(), isConvertPathToLowerCase, awsS3AtlasModelVersion) ; + this(metadataNamespace, new HashMap<>(), isConvertPathToLowerCase, awsS3AtlasModelVersion); } public PathExtractorContext(String metadataNamespace, Map knownEntities, boolean isConvertPathToLowerCase, String awsS3AtlasModelVersion) { @@ -71,4 +71,4 @@ public boolean isConvertPathToLowerCase() { public String getAwsS3AtlasModelVersion() { return awsS3AtlasModelVersion; } -} \ No newline at end of file +} diff --git a/common/src/main/java/org/apache/atlas/utils/PropertiesUtil.java b/common/src/main/java/org/apache/atlas/utils/PropertiesUtil.java index e7c4b94480..a3943e4787 100644 --- a/common/src/main/java/org/apache/atlas/utils/PropertiesUtil.java +++ b/common/src/main/java/org/apache/atlas/utils/PropertiesUtil.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,64 +18,41 @@ package org.apache.atlas.utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; -import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer; - /** * Util class for Properties. */ public final class PropertiesUtil extends PropertyPlaceholderConfigurer { - private static Map propertiesMap = new HashMap<>(); - private static Logger logger = LoggerFactory.getLogger(PropertiesUtil.class); - protected List xmlPropertyConfigurer = new ArrayList<>(); - - private PropertiesUtil() { + private static final Logger logger = LoggerFactory.getLogger(PropertiesUtil.class); - } + private static final Map propertiesMap = new HashMap<>(); - @Override - protected void processProperties(ConfigurableListableBeanFactory beanFactory, Properties props) { + private final List xmlPropertyConfigurer = new ArrayList<>(); - Properties sysProps = System.getProperties(); - if (sysProps != null) { - for (String key : sysProps.stringPropertyNames()) { - String value = sysProps.getProperty(key); - if (value != null) { - value = value.trim(); - } - propertiesMap.put(key, value); - } - } - - if (props != null) { - for (String key : props.stringPropertyNames()) { - String value = props.getProperty(key); - if (value != null) { - value = value.trim(); - } - propertiesMap.put(key, value); - } - } - - super.processProperties(beanFactory, props); + private PropertiesUtil() { } public static String getProperty(String key, String defaultValue) { if (key == null) { return null; } + String rtrnVal = propertiesMap.get(key); + if (rtrnVal == null) { rtrnVal = defaultValue; } + return rtrnVal; } @@ -83,6 +60,7 @@ public static String getProperty(String key) { if (key == null) { return null; } + return propertiesMap.get(key); } @@ -90,13 +68,17 @@ public static String[] getPropertyStringList(String key) { if (key == null) { return null; } + String value = propertiesMap.get(key); + if (value != null) { - String[] splitValues = value.split(","); + String[] splitValues = value.split(","); String[] returnValues = new String[splitValues.length]; + for (int i = 0; i < splitValues.length; i++) { returnValues[i] = splitValues[i].trim(); } + return returnValues; } else { return new String[0]; @@ -107,10 +89,13 @@ public static Integer getIntProperty(String key, int defaultValue) { if (key == null) { return null; } + String rtrnVal = propertiesMap.get(key); + if (rtrnVal == null) { return defaultValue; } + return Integer.valueOf(rtrnVal); } @@ -118,10 +103,13 @@ public static Integer getIntProperty(String key) { if (key == null) { return null; } + String rtrnVal = propertiesMap.get(key); + if (rtrnVal == null) { return null; } + return Integer.valueOf(rtrnVal); } @@ -129,10 +117,44 @@ public static boolean getBooleanProperty(String key, boolean defaultValue) { if (key == null) { return defaultValue; } + String value = getProperty(key); + if (value == null) { return defaultValue; } + return Boolean.parseBoolean(value); } + + @Override + protected void processProperties(ConfigurableListableBeanFactory beanFactory, Properties props) { + Properties sysProps = System.getProperties(); + + if (sysProps != null) { + for (String key : sysProps.stringPropertyNames()) { + String value = sysProps.getProperty(key); + + if (value != null) { + value = value.trim(); + } + + propertiesMap.put(key, value); + } + } + + if (props != null) { + for (String key : props.stringPropertyNames()) { + String value = props.getProperty(key); + + if (value != null) { + value = value.trim(); + } + + propertiesMap.put(key, value); + } + } + + super.processProperties(beanFactory, props); + } } diff --git a/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java b/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java index 0b570685a1..9944dec8d3 100644 --- a/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java +++ b/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,24 +18,21 @@ package org.apache.atlas.utils; - import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -public class SHA256Utils{ - +public class SHA256Utils { + private static final ThreadLocal DIGESTER_FACTORY = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + }); - private static final ThreadLocal DIGESTER_FACTORY = - new ThreadLocal() { - @Override - protected MessageDigest initialValue() { - try { - return MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - }; + private SHA256Utils() { + // to block instanations + } /** * Create a thread local SHA256 digester. @@ -43,26 +40,29 @@ protected MessageDigest initialValue() { public static MessageDigest getDigester() { MessageDigest digester = DIGESTER_FACTORY.get(); + digester.reset(); + return digester; } public static String toString(byte[] hash) { - try{ - StringBuffer hexString = new StringBuffer(); + try { + StringBuilder hexString = new StringBuilder(); for (int i = 0; i < hash.length; i++) { String hex = Integer.toHexString(0xff & hash[i]); - if(hex.length() == 1) hexString.append('0'); + + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); } return hexString.toString(); - } catch(Exception ex){ + } catch (Exception ex) { throw new RuntimeException(ex); } } - - - } diff --git a/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java b/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java index b1188f9140..66bd2bb342 100644 --- a/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java +++ b/common/src/test/java/org/apache/atlas/ha/HAConfigurationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,8 +34,7 @@ import static org.testng.Assert.assertTrue; public class HAConfigurationTest { - - private static final String[] TEST_ATLAS_SERVER_IDS_HA = new String[] { "id1", "id2" }; + private static final String[] TEST_ATLAS_SERVER_IDS_HA = new String[] {"id1", "id2"}; @Mock private Configuration configuration; @@ -43,6 +42,7 @@ public class HAConfigurationTest { @BeforeMethod public void setup() { MockitoAnnotations.initMocks(this); + System.setProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT, AtlasConstants.DEFAULT_APP_PORT_STR); } @@ -68,14 +68,14 @@ public void testIsHAEnabledByIds() { assertTrue(isHAEnabled); // restore - when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(new String[] { "id1"}); + when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(new String[] {"id1"}); isHAEnabled = HAConfiguration.isHAEnabled(configuration); assertFalse(isHAEnabled); } @Test public void testShouldReturnHTTPSBoundAddress() { - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX +"id1")).thenReturn("127.0.0.1:21443"); + when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id1")).thenReturn("127.0.0.1:21443"); when(configuration.getBoolean(SecurityProperties.TLS_ENABLED)).thenReturn(true); String address = HAConfiguration.getBoundAddressForId(configuration, "id1"); @@ -86,8 +86,8 @@ public void testShouldReturnHTTPSBoundAddress() { @Test public void testShouldReturnListOfAddressesInConfig() { when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(TEST_ATLAS_SERVER_IDS_HA); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX +"id1")).thenReturn("127.0.0.1:21000"); - when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX +"id2")).thenReturn("127.0.0.1:31000"); + when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id1")).thenReturn("127.0.0.1:21000"); + when(configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + "id2")).thenReturn("127.0.0.1:31000"); List serverInstances = HAConfiguration.getServerInstances(configuration); assertEquals(serverInstances.size(), 2); @@ -99,8 +99,7 @@ public void testShouldReturnListOfAddressesInConfig() { public void testShouldGetZookeeperAcl() { when(configuration.getString(HAConfiguration.HA_ZOOKEEPER_ACL)).thenReturn("sasl:myclient@EXAMPLE.COM"); - HAConfiguration.ZookeeperProperties zookeeperProperties = - HAConfiguration.getZookeeperProperties(configuration); + HAConfiguration.ZookeeperProperties zookeeperProperties = HAConfiguration.getZookeeperProperties(configuration); assertTrue(zookeeperProperties.hasAcl()); } @@ -108,8 +107,7 @@ public void testShouldGetZookeeperAcl() { public void testShouldGetZookeeperAuth() { when(configuration.getString(HAConfiguration.HA_ZOOKEEPER_AUTH)).thenReturn("sasl:myclient@EXAMPLE.COM"); - HAConfiguration.ZookeeperProperties zookeeperProperties = - HAConfiguration.getZookeeperProperties(configuration); + HAConfiguration.ZookeeperProperties zookeeperProperties = HAConfiguration.getZookeeperProperties(configuration); assertTrue(zookeeperProperties.hasAuth()); } } diff --git a/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java b/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java index c232a7ac87..6b95fd30c3 100644 --- a/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java +++ b/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java @@ -20,115 +20,82 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import org.apache.hadoop.fs.Path; import java.util.HashMap; import java.util.Map; -import static org.testng.Assert.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; public class AtlasPathExtractorUtilTest { private static final Logger LOG = LoggerFactory.getLogger(AtlasPathExtractorUtilTest.class); // Common - private static final String METADATA_NAMESPACE = "metaspace"; - private static final String QNAME_METADATA_NAMESPACE = '@' + METADATA_NAMESPACE; - private static final String SCHEME_SEPARATOR = "://"; - private static final String ATTRIBUTE_NAME = "name"; - private static final String ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; + private static final String METADATA_NAMESPACE = "metaspace"; + private static final String QNAME_METADATA_NAMESPACE = '@' + METADATA_NAMESPACE; + private static final String SCHEME_SEPARATOR = "://"; + private static final String ATTRIBUTE_NAME = "name"; + private static final String ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; // HDFS - private static final String HDFS_PATH_TYPE = "hdfs_path"; - private static final String ATTRIBUTE_PATH = "path"; - private static final String ATTRIBUTE_CLUSTER_NAME = "clusterName"; + private static final String HDFS_PATH_TYPE = "hdfs_path"; + private static final String ATTRIBUTE_PATH = "path"; + private static final String ATTRIBUTE_CLUSTER_NAME = "clusterName"; // Ozone - private static final String OZONE_VOLUME = "ozone_volume"; - private static final String OZONE_BUCKET = "ozone_bucket"; - private static final String OZONE_KEY = "ozone_key"; - private static final String OZONE_SCHEME = "ofs" + SCHEME_SEPARATOR; - private static final String OZONE_3_SCHEME = "o3fs" + SCHEME_SEPARATOR; + private static final String OZONE_VOLUME = "ozone_volume"; + private static final String OZONE_BUCKET = "ozone_bucket"; + private static final String OZONE_KEY = "ozone_key"; + private static final String OZONE_SCHEME = "ofs" + SCHEME_SEPARATOR; + private static final String OZONE_3_SCHEME = "o3fs" + SCHEME_SEPARATOR; // HDFS - private static final String HDFS_SCHEME = "hdfs" + SCHEME_SEPARATOR; - private static final String HDFS_PATH = HDFS_SCHEME + "host_name:8020/warehouse/tablespace/external/hive/taBlE_306"; + private static final String HDFS_SCHEME = "hdfs" + SCHEME_SEPARATOR; + private static final String HDFS_PATH = HDFS_SCHEME + "host_name:8020/warehouse/tablespace/external/hive/taBlE_306"; // ADLS Gen2 - private static final String ADLS_GEN2_ACCOUNT = "adls_gen2_account"; - private static final String ADLS_GEN2_CONTAINER = "adls_gen2_container"; - private static final String ADLS_GEN2_DIRECTORY = "adls_gen2_directory"; - private static final String ABFS_SCHEME = "abfs" + SCHEME_SEPARATOR; - private static final String ABFSS_SCHEME = "abfss" + SCHEME_SEPARATOR; - private static final String ABFS_PATH = ABFS_SCHEME + "data@razrangersan.dfs.core.windows.net/tmp/cdp-demo/sample.csv"; - private static final String ABFSS_PATH = ABFSS_SCHEME + "data@razrangersan.dfs.core.windows.net/tmp/cdp-demo/sample.csv"; + private static final String ADLS_GEN2_ACCOUNT = "adls_gen2_account"; + private static final String ADLS_GEN2_CONTAINER = "adls_gen2_container"; + private static final String ADLS_GEN2_DIRECTORY = "adls_gen2_directory"; + private static final String ABFS_SCHEME = "abfs" + SCHEME_SEPARATOR; + private static final String ABFS_PATH = ABFS_SCHEME + "data@razrangersan.dfs.core.windows.net/tmp/cdp-demo/sample.csv"; + private static final String ABFSS_SCHEME = "abfss" + SCHEME_SEPARATOR; + private static final String ABFSS_PATH = ABFSS_SCHEME + "data@razrangersan.dfs.core.windows.net/tmp/cdp-demo/sample.csv"; // AWS S3 - private static final String AWS_S3_ATLAS_MODEL_VERSION_V2 = "V2"; - private static final String AWS_S3_BUCKET = "aws_s3_bucket"; - private static final String AWS_S3_PSEUDO_DIR = "aws_s3_pseudo_dir"; - private static final String AWS_S3_V2_BUCKET = "aws_s3_v2_bucket"; - private static final String AWS_S3_V2_PSEUDO_DIR = "aws_s3_v2_directory"; - private static final String S3_SCHEME = "s3" + SCHEME_SEPARATOR; - private static final String S3A_SCHEME = "s3a" + SCHEME_SEPARATOR; - private static final String ATTRIBUTE_OBJECT_PREFIX = "objectPrefix"; - private static final String S3_PATH = S3_SCHEME + "aws_my_bucket1/1234567890/renders/Irradiance_A.csv"; - private static final String S3A_PATH = S3A_SCHEME + "aws_my_bucket1/1234567890/renders/Irradiance_A.csv"; - + private static final String AWS_S3_ATLAS_MODEL_VERSION_V2 = "V2"; + private static final String AWS_S3_BUCKET = "aws_s3_bucket"; + private static final String AWS_S3_PSEUDO_DIR = "aws_s3_pseudo_dir"; + private static final String AWS_S3_V2_BUCKET = "aws_s3_v2_bucket"; + private static final String AWS_S3_V2_PSEUDO_DIR = "aws_s3_v2_directory"; + private static final String S3_SCHEME = "s3" + SCHEME_SEPARATOR; + private static final String S3_PATH = S3_SCHEME + "aws_my_bucket1/1234567890/renders/Irradiance_A.csv"; + private static final String S3A_SCHEME = "s3a" + SCHEME_SEPARATOR; + private static final String S3A_PATH = S3A_SCHEME + "aws_my_bucket1/1234567890/renders/Irradiance_A.csv"; + private static final String ATTRIBUTE_OBJECT_PREFIX = "objectPrefix"; // Google Cloud Storage - private static final String GCS_VIRTUAL_DIR = "gcp_storage_virtual_directory"; - private static final String GCS_BUCKET = "gcp_storage_bucket"; - private static final String GCS_SCHEME = "gs" + SCHEME_SEPARATOR; - private static final String GCS_PATH = GCS_SCHEME + "gcs_test_bucket1/1234567890/data"; - - @DataProvider(name = "ozonePathProvider") - private Object[][] ozonePathProvider(){ - return new Object[][]{ - { new OzoneKeyValidator(OZONE_SCHEME, "ozone1.com/volume1/bucket1/files/file.txt", - "files", "ozone1.com/volume1/bucket1/files", - "file.txt", "ozone1.com/volume1/bucket1/files/file.txt")}, - - { new OzoneKeyValidator(OZONE_SCHEME, "ozone1:1234/volume1/bucket1/file21.txt", - "file21.txt", "ozone1:1234/volume1/bucket1/file21.txt")}, - - { new OzoneKeyValidator(OZONE_SCHEME, "ozone1/volume1/bucket1/quarter_one/sales", - "quarter_one", "ozone1/volume1/bucket1/quarter_one", - "sales", "ozone1/volume1/bucket1/quarter_one/sales")}, - - { new OzoneKeyValidator(OZONE_SCHEME, "ozone1/volume1/bucket1/quarter_one/sales/", - "quarter_one", "ozone1/volume1/bucket1/quarter_one", - "sales", "ozone1/volume1/bucket1/quarter_one/sales")}, - - { new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/files/file.txt", - "files", "bucket1.volume1.ozone1/files", - "file.txt", "bucket1.volume1.ozone1/files/file.txt") }, - - { new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/file21.txt", - "file21.txt", "bucket1.volume1.ozone1/file21.txt") }, - - { new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales", - "quarter_one", "bucket1.volume1.ozone1/quarter_one", - "sales", "bucket1.volume1.ozone1/quarter_one/sales") }, - - { new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/", - "quarter_one", "bucket1.volume1.ozone1/quarter_one", - "sales", "bucket1.volume1.ozone1/quarter_one/sales") }, - }; - } + private static final String GCS_VIRTUAL_DIR = "gcp_storage_virtual_directory"; + private static final String GCS_BUCKET = "gcp_storage_bucket"; + private static final String GCS_SCHEME = "gs" + SCHEME_SEPARATOR; + private static final String GCS_PATH = GCS_SCHEME + "gcs_test_bucket1/1234567890/data"; @Test(dataProvider = "ozonePathProvider") public void testGetPathEntityOzone3Path(OzoneKeyValidator validator) { - String scheme = validator.scheme; + String scheme = validator.scheme; String ozonePath = scheme + validator.location; PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); - Path path = new Path(ozonePath); + Path path = new Path(ozonePath); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); verifyOzoneKeyEntity(entity, validator); @@ -144,9 +111,9 @@ public void testGetPathEntityOzone3Path(OzoneKeyValidator validator) { public void testGetPathEntityHdfsPath() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); - Path path = new Path(HDFS_PATH); + Path path = new Path(HDFS_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), HDFS_PATH_TYPE); @@ -161,9 +128,9 @@ public void testGetPathEntityHdfsPath() { public void testGetPathEntityHdfsPathLowerCase() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE, true, null); - Path path = new Path(HDFS_PATH); + Path path = new Path(HDFS_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), HDFS_PATH_TYPE); @@ -178,9 +145,9 @@ public void testGetPathEntityHdfsPathLowerCase() { public void testGetPathEntityABFSPath() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); - Path path = new Path(ABFS_PATH); + Path path = new Path(ABFS_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), ADLS_GEN2_DIRECTORY); @@ -194,9 +161,9 @@ public void testGetPathEntityABFSPath() { public void testGetPathEntityABFSSPath() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); - Path path = new Path(ABFSS_PATH); + Path path = new Path(ABFSS_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), ADLS_GEN2_DIRECTORY); @@ -210,9 +177,9 @@ public void testGetPathEntityABFSSPath() { public void testGetPathEntityS3V2Path() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE, AWS_S3_ATLAS_MODEL_VERSION_V2); - Path path = new Path(S3_PATH); + Path path = new Path(S3_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), AWS_S3_V2_PSEUDO_DIR); @@ -226,9 +193,9 @@ public void testGetPathEntityS3V2Path() { public void testGetPathEntityS3AV2Path() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE, AWS_S3_ATLAS_MODEL_VERSION_V2); - Path path = new Path(S3A_PATH); + Path path = new Path(S3A_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), AWS_S3_V2_PSEUDO_DIR); @@ -242,9 +209,9 @@ public void testGetPathEntityS3AV2Path() { public void testGetPathEntityS3Path() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); - Path path = new Path(S3_PATH); + Path path = new Path(S3_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), AWS_S3_PSEUDO_DIR); @@ -258,9 +225,9 @@ public void testGetPathEntityS3Path() { public void testGetPathEntityS3APath() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); - Path path = new Path(S3A_PATH); + Path path = new Path(S3A_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), AWS_S3_PSEUDO_DIR); @@ -274,9 +241,9 @@ public void testGetPathEntityS3APath() { public void testGetPathEntityGCSPath() { PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE); - Path path = new Path(GCS_PATH); + Path path = new Path(GCS_PATH); AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext); - AtlasEntity entity = entityWithExtInfo.getEntity(); + AtlasEntity entity = entityWithExtInfo.getEntity(); assertNotNull(entity); assertEquals(entity.getTypeName(), GCS_VIRTUAL_DIR); @@ -286,9 +253,44 @@ public void testGetPathEntityGCSPath() { verifyGCSKnownEntities(GCS_SCHEME, GCS_PATH, extractorContext.getKnownEntities()); } + @DataProvider(name = "ozonePathProvider") + private Object[][] ozonePathProvider() { + return new Object[][] { + {new OzoneKeyValidator(OZONE_SCHEME, "ozone1.com/volume1/bucket1/files/file.txt", + "files", "ozone1.com/volume1/bucket1/files", + "file.txt", "ozone1.com/volume1/bucket1/files/file.txt")}, + + {new OzoneKeyValidator(OZONE_SCHEME, "ozone1:1234/volume1/bucket1/file21.txt", + "file21.txt", "ozone1:1234/volume1/bucket1/file21.txt")}, + + {new OzoneKeyValidator(OZONE_SCHEME, "ozone1/volume1/bucket1/quarter_one/sales", + "quarter_one", "ozone1/volume1/bucket1/quarter_one", + "sales", "ozone1/volume1/bucket1/quarter_one/sales")}, + + {new OzoneKeyValidator(OZONE_SCHEME, "ozone1/volume1/bucket1/quarter_one/sales/", + "quarter_one", "ozone1/volume1/bucket1/quarter_one", + "sales", "ozone1/volume1/bucket1/quarter_one/sales")}, + + {new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/files/file.txt", + "files", "bucket1.volume1.ozone1/files", + "file.txt", "bucket1.volume1.ozone1/files/file.txt")}, + + {new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/file21.txt", + "file21.txt", "bucket1.volume1.ozone1/file21.txt")}, + + {new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales", + "quarter_one", "bucket1.volume1.ozone1/quarter_one", + "sales", "bucket1.volume1.ozone1/quarter_one/sales")}, + + {new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/", + "quarter_one", "bucket1.volume1.ozone1/quarter_one", + "sales", "bucket1.volume1.ozone1/quarter_one/sales")}, + }; + } + private void verifyOzoneEntities(Map knownEntities, OzoneKeyValidator validator) { for (AtlasEntity knownEntity : knownEntities.values()) { - switch (knownEntity.getTypeName()){ + switch (knownEntity.getTypeName()) { case OZONE_KEY: verifyOzoneKeyEntity(knownEntity, validator); break; @@ -325,15 +327,15 @@ private void verifyHDFSEntity(AtlasEntity entity, boolean toLowerCase) { } } - private void verifyABFSAdlsGen2Dir(String abfsScheme, String path, AtlasEntity entity){ - String pathQName = abfsScheme + "data@razrangersan/tmp/cdp-demo/sample.csv" + QNAME_METADATA_NAMESPACE; + private void verifyABFSAdlsGen2Dir(String abfsScheme, String path, AtlasEntity entity) { + String pathQName = abfsScheme + "data@razrangersan/tmp/cdp-demo/sample.csv" + QNAME_METADATA_NAMESPACE; String entityQName = (String) entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME); - if (pathQName.equalsIgnoreCase(entityQName)){ + if (pathQName.equalsIgnoreCase(entityQName)) { assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "sample.csv"); } else { pathQName = abfsScheme + "data@razrangersan/tmp/cdp-demo" + QNAME_METADATA_NAMESPACE; - if (pathQName.equalsIgnoreCase(entityQName)){ + if (pathQName.equalsIgnoreCase(entityQName)) { assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "cdp-demo"); } else { assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), abfsScheme + "data@razrangersan/tmp" + QNAME_METADATA_NAMESPACE); @@ -346,7 +348,7 @@ private void verifyABFSKnownEntities(String scheme, String path, Map getPairMap(String scheme, String... pairs){ - Map< String, String > ret = new HashMap<>(); + private Map getPairMap(String scheme, String... pairs) { + Map ret = new HashMap<>(); for (int i = 0; i < pairs.length; i += 2) { - ret.put(pairs[i], scheme + pairs[i+1] + QNAME_METADATA_NAMESPACE); + ret.put(pairs[i], scheme + pairs[i + 1] + QNAME_METADATA_NAMESPACE); } return ret; diff --git a/common/src/test/java/org/apache/atlas/utils/HdfsNameServiceResolverTest.java b/common/src/test/java/org/apache/atlas/utils/HdfsNameServiceResolverTest.java index 30bc2e1a32..9dc369e837 100644 --- a/common/src/test/java/org/apache/atlas/utils/HdfsNameServiceResolverTest.java +++ b/common/src/test/java/org/apache/atlas/utils/HdfsNameServiceResolverTest.java @@ -22,7 +22,6 @@ import static org.testng.Assert.assertEquals; public class HdfsNameServiceResolverTest { - @Test public void testResolution() { // assertEquals(HdfsNameServiceResolver.getNameServiceID("test"), ""); @@ -50,6 +49,5 @@ public void testResolution() { assertEquals(HdfsNameServiceResolver.getPathWithNameServiceID("hdfs://mycluster/tmp/xyz"), "hdfs://mycluster/tmp/xyz"); assertEquals(HdfsNameServiceResolver.getNameServiceIDForPath("hdfs://mycluster/tmp/xyz"), "mycluster"); - } -} \ No newline at end of file +} diff --git a/common/src/test/java/org/apache/atlas/utils/KafkaUtilsTest.java b/common/src/test/java/org/apache/atlas/utils/KafkaUtilsTest.java index 9b4f093e71..dc6dc48255 100644 --- a/common/src/test/java/org/apache/atlas/utils/KafkaUtilsTest.java +++ b/common/src/test/java/org/apache/atlas/utils/KafkaUtilsTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,6 @@ import org.mockito.Mockito; import org.testng.annotations.Test; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.util.HashMap; @@ -43,142 +42,139 @@ import static org.testng.Assert.fail; public class KafkaUtilsTest { + protected static final String JAAS_MASKED_PASSWORD = "keypass"; - protected Path jksPath; + protected Path jksPath; protected String providerUrl; - protected static final String JAAS_MASKED_PASSWORD = "keypass"; - @Test public void testSetKafkaJAASPropertiesForAllProperValues() { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); - final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; + final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "true"; - final String optionServiceName = "kafka"; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "true"; + final String optionServiceName = "kafka"; - configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.KafkaClient.option.storeKey", optionStoreKey); - configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName",optionServiceName); + configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName", optionServiceName); KafkaUtils.setKafkaJAASProperties(configuration, properties); + String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); - assertTrue(newPropertyValue.contains(loginModuleControlFlag),"loginModuleControlFlag not present in new property"); + assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("useKeyTab=\"" + optionUseKeyTab + "\""), "useKeyTab not present in new property or value doesn't match"); - assertTrue(newPropertyValue.contains("storeKey=\""+ optionStoreKey + "\""), "storeKey not present in new property or value doesn't match"); + assertTrue(newPropertyValue.contains("storeKey=\"" + optionStoreKey + "\""), "storeKey not present in new property or value doesn't match"); assertTrue(newPropertyValue.contains("serviceName=\"" + optionServiceName + "\""), "serviceName not present in new property or value doesn't match"); assertJaaSConfigLoadable(newPropertyValue); - } @Test public void testSetKafkaJAASPropertiesForMissingControlFlag() { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); - final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; + final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "true"; - final String optionServiceName = "kafka"; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "true"; + final String optionServiceName = "kafka"; - configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.KafkaClient.option.storeKey", optionStoreKey); - configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName",optionServiceName); + configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName", optionServiceName); KafkaUtils.setKafkaJAASProperties(configuration, properties); + String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); - assertTrue(newPropertyValue.contains(loginModuleControlFlag),"loginModuleControlFlag not present in new property"); + assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("useKeyTab=\"" + optionUseKeyTab + "\""), "useKeyTab not present in new property or value doesn't match"); - assertTrue(newPropertyValue.contains("storeKey=\""+ optionStoreKey + "\""), "storeKey not present in new property or value doesn't match"); + assertTrue(newPropertyValue.contains("storeKey=\"" + optionStoreKey + "\""), "storeKey not present in new property or value doesn't match"); assertTrue(newPropertyValue.contains("serviceName=\"" + optionServiceName + "\""), "serviceName not present in new property or value doesn't match"); assertJaaSConfigLoadable(newPropertyValue); - } @Test public void testSetKafkaJAASPropertiesForMissingLoginModuleName() { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "true"; - final String optionServiceName = "kafka"; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "true"; + final String optionServiceName = "kafka"; configuration.setProperty("atlas.jaas.KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.KafkaClient.option.storeKey", optionStoreKey); - configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName",optionServiceName); + configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName", optionServiceName); KafkaUtils.setKafkaJAASProperties(configuration, properties); + String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); assertNull(newPropertyValue); - } @Test public void testSetKafkaJAASPropertiesWithSpecialCharacters() { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); - final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; + final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; final String loginModuleControlFlag = "required"; - final String optionKeyTabPath = "/path/to/file.keytab"; - final String optionPrincipal = "test/_HOST@EXAMPLE.COM"; + final String optionKeyTabPath = "/path/to/file.keytab"; + final String optionPrincipal = "test/_HOST@EXAMPLE.COM"; - configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.KafkaClient.option.keyTabPath", optionKeyTabPath); configuration.setProperty("atlas.jaas.KafkaClient.option.principal", optionPrincipal); try { KafkaUtils.setKafkaJAASProperties(configuration, properties); - String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); + String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); String updatedPrincipalValue = org.apache.hadoop.security.SecurityUtil.getServerPrincipal(optionPrincipal, (String) null); assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); - assertTrue(newPropertyValue.contains(loginModuleControlFlag),"loginModuleControlFlag not present in new property"); + assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("keyTabPath=\"" + optionKeyTabPath + "\"")); - assertTrue(newPropertyValue.contains("principal=\""+ updatedPrincipalValue + "\"")); + assertTrue(newPropertyValue.contains("principal=\"" + updatedPrincipalValue + "\"")); assertJaaSConfigLoadable(newPropertyValue); - } catch (IOException e) { fail("Failed while getting updated principal value with exception : " + e.getMessage()); } - } @Test public void testSetKafkaJAASPropertiesForTicketBasedLoginConfig() { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); - final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; + final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "true"; - final String optionServiceName = "kafka"; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "true"; + final String optionServiceName = "kafka"; configuration.setProperty("atlas.kafka.bootstrap.servers", "localhost:9100"); - configuration.setProperty("atlas.jaas.ticketBased-KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.ticketBased-KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.ticketBased-KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.ticketBased-KafkaClient.option.storeKey", optionStoreKey); - configuration.setProperty("atlas.jaas.ticketBased-KafkaClient.option.serviceName",optionServiceName); + configuration.setProperty("atlas.jaas.ticketBased-KafkaClient.option.serviceName", optionServiceName); - try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { + try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { mockedKafkaUtilsClass.when(KafkaUtils::isLoginKeytabBased).thenReturn(false); mockedKafkaUtilsClass.when(KafkaUtils::isLoginTicketBased).thenReturn(true); mockedKafkaUtilsClass.when(() -> KafkaUtils.surroundWithQuotes(Mockito.anyString())).thenCallRealMethod(); @@ -187,6 +183,7 @@ public void testSetKafkaJAASPropertiesForTicketBasedLoginConfig() { KafkaUtils.setKafkaJAASProperties(configuration, properties); String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); + assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("useKeyTab=\"" + optionUseKeyTab + "\""), "useKeyTab not present in new property or value doesn't match"); @@ -198,22 +195,22 @@ public void testSetKafkaJAASPropertiesForTicketBasedLoginConfig() { @Test public void testSetKafkaJAASPropertiesForTicketBasedLoginFallback() { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); - final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; + final String loginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "true"; - final String optionServiceName = "kafka"; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "true"; + final String optionServiceName = "kafka"; - configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.KafkaClient.option.storeKey", optionStoreKey); - configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName",optionServiceName); + configuration.setProperty("atlas.jaas.KafkaClient.option.serviceName", optionServiceName); - try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { + try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { mockedKafkaUtilsClass.when(KafkaUtils::isLoginKeytabBased).thenReturn(false); mockedKafkaUtilsClass.when(KafkaUtils::isLoginTicketBased).thenReturn(true); mockedKafkaUtilsClass.when(() -> KafkaUtils.surroundWithQuotes(Mockito.anyString())).thenCallRealMethod(); @@ -222,6 +219,7 @@ public void testSetKafkaJAASPropertiesForTicketBasedLoginFallback() { KafkaUtils.setKafkaJAASProperties(configuration, properties); String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); + assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("useKeyTab=\"" + optionUseKeyTab + "\""), "useKeyTab not present in new property or value doesn't match"); @@ -233,20 +231,20 @@ public void testSetKafkaJAASPropertiesForTicketBasedLoginFallback() { @Test public void testSetKafkaJAASPropertiesForTokenAuthConfig() { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); - final String loginModuleName = "org.apache.kafka.common.security.scram.ScramLoginModule"; + final String loginModuleName = "org.apache.kafka.common.security.scram.ScramLoginModule"; final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "false"; - final String optionServiceName = "kafka"; - final String optionTokenAuth = "true"; - final String optionUsername = "30CQ4q1hQMy0dB6X0eXfxQ"; - final String optionPassword = "KdaUQ4FlKWlDxwQrAeFGUVbb6sR0P+zoqOZDZjtIRP1wseXbSbhiTjz3QI9Ur9o4LTYZSv8TE1QqUC4FSwnoTA=="; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "false"; + final String optionServiceName = "kafka"; + final String optionTokenAuth = "true"; + final String optionUsername = "30CQ4q1hQMy0dB6X0eXfxQ"; + final String optionPassword = "KdaUQ4FlKWlDxwQrAeFGUVbb6sR0P+zoqOZDZjtIRP1wseXbSbhiTjz3QI9Ur9o4LTYZSv8TE1QqUC4FSwnoTA=="; configuration.setProperty("atlas.kafka.bootstrap.servers", "localhost:9100"); - configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.KafkaClient.option.storeKey", optionStoreKey); @@ -255,13 +253,14 @@ public void testSetKafkaJAASPropertiesForTokenAuthConfig() { configuration.setProperty("atlas.jaas.KafkaClient.option.username", optionUsername); configuration.setProperty("atlas.jaas.KafkaClient.option.password", optionPassword); - try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { + try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { mockedKafkaUtilsClass.when(() -> KafkaUtils.surroundWithQuotes(Mockito.anyString())).thenCallRealMethod(); mockedKafkaUtilsClass.when(() -> KafkaUtils.setKafkaJAASProperties(configuration, properties)).thenCallRealMethod(); KafkaUtils.setKafkaJAASProperties(configuration, properties); String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); + assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("useKeyTab=\"" + optionUseKeyTab + "\""), "useKeyTab not present in new property or value doesn't match"); @@ -276,20 +275,22 @@ public void testSetKafkaJAASPropertiesForTokenAuthConfig() { @Test public void testSetKafkaJAASPropertiesForClearTextPassword() throws Exception { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); + setupCredentials(); - final String loginModuleName = "org.apache.kafka.common.security.scram.ScramLoginModule"; + + final String loginModuleName = "org.apache.kafka.common.security.scram.ScramLoginModule"; final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "false"; - final String optionServiceName = "kafka"; - final String optionTokenAuth = "true"; - final String optionUsername = "30CQ4q1hQMy0dB6X0eXfxQ"; - final String optionPassword = "admin123"; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "false"; + final String optionServiceName = "kafka"; + final String optionTokenAuth = "true"; + final String optionUsername = "30CQ4q1hQMy0dB6X0eXfxQ"; + final String optionPassword = "admin123"; configuration.setProperty("atlas.kafka.bootstrap.servers", "localhost:9100"); - configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.KafkaClient.option.storeKey", optionStoreKey); @@ -299,13 +300,14 @@ public void testSetKafkaJAASPropertiesForClearTextPassword() throws Exception { configuration.setProperty("atlas.jaas.KafkaClient.option.password", optionPassword); configuration.setProperty("hadoop.security.credential.provider.path", providerUrl); - try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { + try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { mockedKafkaUtilsClass.when(() -> KafkaUtils.surroundWithQuotes(Mockito.anyString())).thenCallRealMethod(); mockedKafkaUtilsClass.when(() -> KafkaUtils.setKafkaJAASProperties(configuration, properties)).thenCallRealMethod(); KafkaUtils.setKafkaJAASProperties(configuration, properties); String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); + assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("useKeyTab=\"" + optionUseKeyTab + "\""), "useKeyTab not present in new property or value doesn't match"); @@ -320,20 +322,22 @@ public void testSetKafkaJAASPropertiesForClearTextPassword() throws Exception { @Test public void testSetKafkaJAASPropertiesForPasswordEncryption() throws Exception { - Properties properties = new Properties(); + Properties properties = new Properties(); Configuration configuration = new PropertiesConfiguration(); + setupCredentials(); - final String loginModuleName = "org.apache.kafka.common.security.scram.ScramLoginModule"; + + final String loginModuleName = "org.apache.kafka.common.security.scram.ScramLoginModule"; final String loginModuleControlFlag = "required"; - final String optionUseKeyTab = "false"; - final String optionStoreKey = "false"; - final String optionServiceName = "kafka"; - final String optionTokenAuth = "true"; - final String optionUsername = "30CQ4q1hQMy0dB6X0eXfxQ"; - final String optionPassword = "********"; + final String optionUseKeyTab = "false"; + final String optionStoreKey = "false"; + final String optionServiceName = "kafka"; + final String optionTokenAuth = "true"; + final String optionUsername = "30CQ4q1hQMy0dB6X0eXfxQ"; + final String optionPassword = "********"; configuration.setProperty("atlas.kafka.bootstrap.servers", "localhost:9100"); - configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName",loginModuleName); + configuration.setProperty("atlas.jaas.KafkaClient.loginModuleName", loginModuleName); configuration.setProperty("atlas.jaas.KafkaClient.loginModuleControlFlag", loginModuleControlFlag); configuration.setProperty("atlas.jaas.KafkaClient.option.useKeyTab", optionUseKeyTab); configuration.setProperty("atlas.jaas.KafkaClient.option.storeKey", optionStoreKey); @@ -343,13 +347,14 @@ public void testSetKafkaJAASPropertiesForPasswordEncryption() throws Exception { configuration.setProperty("atlas.jaas.KafkaClient.option.password", optionPassword); configuration.setProperty("hadoop.security.credential.provider.path", providerUrl); - try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { + try (MockedStatic mockedKafkaUtilsClass = Mockito.mockStatic(KafkaUtils.class)) { mockedKafkaUtilsClass.when(() -> KafkaUtils.surroundWithQuotes(Mockito.anyString())).thenCallRealMethod(); mockedKafkaUtilsClass.when(() -> KafkaUtils.setKafkaJAASProperties(configuration, properties)).thenCallRealMethod(); KafkaUtils.setKafkaJAASProperties(configuration, properties); String newPropertyValue = properties.getProperty(KafkaUtils.KAFKA_SASL_JAAS_CONFIG_PROPERTY); + assertTrue(newPropertyValue.contains(loginModuleName), "loginModuleName not present in new property"); assertTrue(newPropertyValue.contains(loginModuleControlFlag), "loginModuleControlFlag not present in new property"); assertTrue(newPropertyValue.contains("useKeyTab=\"" + optionUseKeyTab + "\""), "useKeyTab not present in new property or value doesn't match"); @@ -362,20 +367,10 @@ public void testSetKafkaJAASPropertiesForPasswordEncryption() throws Exception { } } - private void assertJaaSConfigLoadable(String jaasConfig) { - // Ensure that JaaS config can be loaded - Map jaasConfigs = new HashMap<>(); - jaasConfigs.put(SaslConfigs.SASL_JAAS_CONFIG, new Password(jaasConfig)); - try { - JaasContext.loadClientContext(jaasConfigs); - } catch (IllegalArgumentException e) { - fail(String.format("JaaS config '%s' can not be loaded", jaasConfig), e); - } - } - protected void setupCredentials() throws Exception { - jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "kafka.jceks"); + jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "kafka.jceks"); providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file/" + jksPath.toUri(); + org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(false); conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl); @@ -386,7 +381,6 @@ protected void setupCredentials() throws Exception { try { provider.createCredentialEntry("atlas.jaas.KafkaClient.option.password", JAAS_MASKED_PASSWORD.toCharArray()); - // write out so that it can be found in checks provider.flush(); } catch (Exception e) { @@ -395,5 +389,16 @@ protected void setupCredentials() throws Exception { } } + private void assertJaaSConfigLoadable(String jaasConfig) { + // Ensure that JaaS config can be loaded + Map jaasConfigs = new HashMap<>(); + + jaasConfigs.put(SaslConfigs.SASL_JAAS_CONFIG, new Password(jaasConfig)); + try { + JaasContext.loadClientContext(jaasConfigs); + } catch (IllegalArgumentException e) { + fail(String.format("JaaS config '%s' can not be loaded", jaasConfig), e); + } + } } diff --git a/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java b/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java index 24d62f5b4d..72cb12c540 100644 --- a/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java +++ b/common/src/test/java/org/apache/atlas/utils/LruCacheTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,29 +17,27 @@ */ package org.apache.atlas.utils; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; +import org.apache.commons.lang.RandomStringUtils; +import org.testng.annotations.Test; import java.util.HashMap; import java.util.Map; -import org.apache.commons.lang.RandomStringUtils; -import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; /** * Tests the LruCache. */ @Test public class LruCacheTest { - /** * Tests the basic operations on the cache. */ @Test - public void testBasicOps() throws Exception { - + public void testBasicOps() { LruCache cache = new LruCache<>(1000, 0); // Get the static cache and populate it. Its size and other // characteristics depend on the bootstrap properties that are hard to @@ -107,8 +105,8 @@ public void testBasicOps() throws Exception { @Test public void testMapOperations() { - Map reference = new HashMap<>(); + reference.put("name", "Fred"); reference.put("occupation", "student"); reference.put("height", "5'11"); @@ -116,29 +114,32 @@ public void testMapOperations() { reference.put("State", "MA"); LruCache map = new LruCache<>(10, 10); + map.putAll(reference); assertEquals(map.size(), reference.size()); - assertEquals(map.keySet().size(), reference.keySet().size()); + assertEquals(map.size(), reference.size()); assertTrue(map.keySet().containsAll(reference.keySet())); assertTrue(reference.keySet().containsAll(map.keySet())); - assertEquals(reference.entrySet().size(), map.entrySet().size()); - for(Map.Entry entry : map.entrySet()) { + assertEquals(reference.size(), map.size()); + + for (Map.Entry entry : map.entrySet()) { assertTrue(reference.containsKey(entry.getKey())); assertEquals(entry.getValue(), reference.get(entry.getKey())); assertTrue(map.containsKey(entry.getKey())); assertTrue(map.containsValue(entry.getValue())); - assertTrue(map.values().contains(entry.getValue())); + assertTrue(map.containsValue(entry.getValue())); } - assertTrue(reference.equals(map)); - assertTrue(map.equals(reference)); + assertEquals(map, reference); + assertEquals(reference, map); } @Test public void testReplaceValueInMap() { LruCache map = new LruCache<>(10, 10); + map.put("name", "Fred"); map.put("name", "George"); @@ -146,17 +147,17 @@ public void testReplaceValueInMap() { assertEquals(map.size(), 1); } - - @Test public void testOrderUpdatedWhenAddExisting() { LruCache map = new LruCache<>(2, 10); + map.put("name", "Fred"); map.put("age", "15"); map.put("name", "George"); //age should be evicted map.put("height", "5'3\""); + //age is now least recently used assertFalse(map.containsKey("age")); } @@ -164,34 +165,32 @@ public void testOrderUpdatedWhenAddExisting() { @Test public void testMapRemove() { LruCache map = new LruCache<>(10, 10); + map.put("name", "Fred"); map.put("occupation", "student"); map.put("height", "5'11"); map.put("City", "Littleton"); map.put("State", "MA"); + assertMapHasSize(map, 5); assertTrue(map.containsKey("State")); + map.remove("State"); + assertMapHasSize(map, 4); assertFalse(map.containsKey("State")); - - } - - private void assertMapHasSize(LruCache map, int size) { - assertEquals(map.size(), size); - assertEquals(map.keySet().size(), size); - assertEquals(map.values().size(), size); - assertEquals(map.entrySet().size(), size); } @Test public void testEvict() { LruCache map = new LruCache<>(5, 10); + map.put("name", "Fred"); map.put("occupation", "student"); map.put("height", "5'11"); map.put("City", "Littleton"); map.put("State", "MA"); + assertMapHasSize(map, 5); //name should be evicted next @@ -208,11 +207,18 @@ public void testEvict() { assertMapHasSize(map, 5); } + private void assertMapHasSize(LruCache map, int size) { + assertEquals(map.size(), size); + assertEquals(map.size(), size); + assertEquals(map.size(), size); + assertEquals(map.size(), size); + } + /** * Create a fake query handle for testing. * - * @param queryPrefix - * @param pkgPrefix + * @param s1 + * @param s2 * @return a new query handle. */ private String createHandle(String s1, String s2) { @@ -223,11 +229,8 @@ private String createHandle(String s1, String s2) { * Create a mock IInternalQuery. * * @return a mock IInternalQuery. - * @throws QueryException */ private String createQuery() { return RandomStringUtils.randomAlphabetic(10); } - - } diff --git a/repository/src/test/java/org/apache/atlas/pc/StatusReporterTest.java b/repository/src/test/java/org/apache/atlas/pc/StatusReporterTest.java index a0c399b441..bfe272f118 100644 --- a/repository/src/test/java/org/apache/atlas/pc/StatusReporterTest.java +++ b/repository/src/test/java/org/apache/atlas/pc/StatusReporterTest.java @@ -28,7 +28,8 @@ public class StatusReporterTest { private static class IntegerConsumer extends WorkItemConsumer { - private static ThreadLocal payload = new ThreadLocal(); + private static final ThreadLocal payload = new ThreadLocal<>(); + private Integer current; public IntegerConsumer(BlockingQueue queue) { @@ -44,7 +45,9 @@ protected void doCommit() { protected void processItem(Integer item) { try { this.current = item; + Thread.sleep(20 + RandomUtils.nextInt(5, 7)); + super.commit(); } catch (InterruptedException e) { e.printStackTrace(); @@ -52,47 +55,52 @@ protected void processItem(Integer item) { } } - private class IntegerConsumerBuilder implements WorkItemBuilder { + private static class IntegerConsumerBuilder implements WorkItemBuilder, Integer> { @Override public IntegerConsumer build(BlockingQueue queue) { return new IntegerConsumer(queue); } } - private WorkItemManager getWorkItemManger(IntegerConsumerBuilder cb, int numWorkers) { - return new WorkItemManager<>(cb, "IntegerConsumer", 5, numWorkers, true); + private WorkItemManager> getWorkItemManger(WorkItemBuilder, Integer> cb, int numWorkers) { + return new WorkItemManager>(cb, "IntegerConsumer", 5, numWorkers, true); } @Test public void statusReporting() throws InterruptedException { - final int maxItems = 50; - - IntegerConsumerBuilder cb = new IntegerConsumerBuilder(); - WorkItemManager wi = getWorkItemManger(cb, 5); - StatusReporter statusReporter = new StatusReporter<>(); + final int maxItems = 50; + WorkItemBuilder, Integer> cb = new IntegerConsumerBuilder(); + WorkItemManager> wi = getWorkItemManger(cb, 5); + StatusReporter statusReporter = new StatusReporter<>(); for (int i = 0; i < maxItems; i++) { wi.produce(i); + statusReporter.produced(i, i); extractResults(wi, statusReporter); } wi.drain(); + extractResults(wi, statusReporter); + assertEquals(statusReporter.ack().intValue(), (maxItems - 1)); + wi.shutdown(); assertEquals(statusReporter.getProducedCount(), 0); assertEquals(statusReporter.getProcessedCount(), 0); } - private void extractResults(WorkItemManager wi, StatusReporter statusReporter) { - Object result = null; + private void extractResults(WorkItemManager> wi, StatusReporter statusReporter) { + Object result; + while((result = wi.getResults().poll()) != null) { - if (result == null || !(result instanceof Integer)) { + if (!(result instanceof Integer)) { continue; } + statusReporter.processed((Integer) result); } } @@ -104,11 +112,15 @@ public void reportWithTimeout() throws InterruptedException { statusReporter.produced(2, 200); statusReporter.processed(2); + Integer ack = statusReporter.ack(); + assertNull(ack); Thread.sleep(3000); + ack = statusReporter.ack(); + assertNotNull(ack); assertEquals(ack, Integer.valueOf(200)); } diff --git a/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerTest.java b/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerTest.java index e72bdd027b..62034eca0a 100644 --- a/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerTest.java +++ b/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerTest.java @@ -32,19 +32,19 @@ public class WorkItemManagerTest { private static final Logger LOG = LoggerFactory.getLogger(WorkItemManagerTest.class); - private class IntegerConsumer extends WorkItemConsumer { - + private static class IntegerConsumer extends WorkItemConsumer { private final ConcurrentLinkedQueue target; public IntegerConsumer(BlockingQueue queue, ConcurrentLinkedQueue target) { super(queue); + this.target = target; } @Override protected void doCommit() { try { - Thread.sleep(20 * RandomUtils.nextInt(10, 15)); + Thread.sleep(20L * RandomUtils.nextInt(10, 15)); } catch (InterruptedException e) { e.printStackTrace(); } @@ -57,7 +57,7 @@ protected void processItem(Integer item) { } } - private class IntegerConsumerBuilder implements WorkItemBuilder { + private static class IntegerConsumerBuilder implements WorkItemBuilder, Integer> { ConcurrentLinkedQueue integers = new ConcurrentLinkedQueue<>(); @Override @@ -71,7 +71,7 @@ public void oneWorkerSequences() { IntegerConsumerBuilder cb = new IntegerConsumerBuilder(); int numberOfItems = 10; try { - WorkItemManager wi = getWorkItemManger(cb, 1); + WorkItemManager> wi = getWorkItemManger(cb, 1); for (int i = 0; i < numberOfItems; i++) { wi.produce(i); } @@ -94,7 +94,7 @@ public void multipleWorkersUnpredictableSequence() { IntegerConsumerBuilder cb = new IntegerConsumerBuilder(); int numberOfItems = 100; try { - WorkItemManager wi = getWorkItemManger(cb, 5); + WorkItemManager> wi = getWorkItemManger(cb, 5); for (int i = 0; i < numberOfItems; i++) { wi.produce(i); } @@ -107,7 +107,7 @@ public void multipleWorkersUnpredictableSequence() { assertEquals(cb.integers.size(), numberOfItems); } - private WorkItemManager getWorkItemManger(IntegerConsumerBuilder cb, int numWorkers) { + private WorkItemManager> getWorkItemManger(IntegerConsumerBuilder cb, int numWorkers) { return new WorkItemManager<>(cb, 5, numWorkers); } } diff --git a/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerWithResultsTest.java b/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerWithResultsTest.java index a12b597289..d6605defea 100644 --- a/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerWithResultsTest.java +++ b/repository/src/test/java/org/apache/atlas/pc/WorkItemManagerWithResultsTest.java @@ -34,7 +34,7 @@ public class WorkItemManagerWithResultsTest { private static final Logger LOG = LoggerFactory.getLogger(WorkItemManagerWithResultsTest.class); private static class IntegerConsumer extends WorkItemConsumer { - private static ThreadLocal payload = new ThreadLocal(); + private static final ThreadLocal payload = new ThreadLocal<>(); public IntegerConsumer(BlockingQueue queue) { super(queue); @@ -44,6 +44,7 @@ public IntegerConsumer(BlockingQueue queue) { protected void doCommit() { if (getPayload() == -1) { LOG.debug("Skipping:"); + return; } @@ -56,8 +57,10 @@ protected void doCommit() { @Override protected void processItem(Integer item) { try { - setPayload(item.intValue()); + setPayload(item); + Thread.sleep(20 + RandomUtils.nextInt(5, 7)); + super.commit(); } catch (InterruptedException e) { e.printStackTrace(); @@ -77,31 +80,33 @@ public void incrementPayload(int v) { } } - private class IntegerConsumerBuilder implements WorkItemBuilder { + private static class IntegerConsumerBuilder implements WorkItemBuilder, Integer> { @Override public IntegerConsumer build(BlockingQueue queue) { return new IntegerConsumer(queue); } } - private WorkItemManager getWorkItemManger(IntegerConsumerBuilder cb, int numWorkers) { + private WorkItemManager> getWorkItemManger(IntegerConsumerBuilder cb, int numWorkers) { return new WorkItemManager<>(cb, "IntegerConsumer", 5, numWorkers, true); } @Test public void drainTest() throws InterruptedException { - final int maxItems = 50; - - IntegerConsumerBuilder cb = new IntegerConsumerBuilder(); - WorkItemManager wi = getWorkItemManger(cb, 5); + final int maxItems = 50; + IntegerConsumerBuilder cb = new IntegerConsumerBuilder(); + WorkItemManager> wi = getWorkItemManger(cb, 5); for (int i = 0; i < maxItems; i++) { wi.produce(i); } wi.drain(); + assertEquals(wi.getResults().size(), maxItems); - Set set = new HashSet(wi.getResults()); + + Set set = new HashSet<>(wi.getResults()); + assertEquals(set.size(), maxItems); wi.shutdown(); @@ -109,8 +114,8 @@ public void drainTest() throws InterruptedException { @Test public void drainCheckProduceTest() throws InterruptedException { - IntegerConsumerBuilder cb = new IntegerConsumerBuilder(); - WorkItemManager wi = getWorkItemManger(cb, 2); + IntegerConsumerBuilder cb = new IntegerConsumerBuilder(); + WorkItemManager> wi = getWorkItemManger(cb, 2); for (int i = 0; i < 5; i++) { repeatedDrainAndProduce(i, wi); @@ -119,19 +124,22 @@ public void drainCheckProduceTest() throws InterruptedException { wi.shutdown(); } - private void repeatedDrainAndProduce(int runCount, WorkItemManager wi) { + private void repeatedDrainAndProduce(int runCount, WorkItemManager> wi) { final int maxItems = 100; - int halfWay = maxItems / 2; + int halfWay = maxItems / 2; LOG.info("Run: {}", runCount); + wi.getResults().clear(); for (int i = 0; i < maxItems; i++) { if (i == halfWay) { wi.drain(); - Set set = new HashSet(wi.getResults()); + Set set = new HashSet<>(wi.getResults()); + assertEquals(wi.getResults().size(), halfWay, "halfWay: total count"); + assertEquals(set.size(), halfWay, "halfWay: set match"); } @@ -139,8 +147,11 @@ private void repeatedDrainAndProduce(int runCount, WorkItemManager set = new HashSet(wi.getResults()); + + Set set = new HashSet<>(wi.getResults()); + assertEquals(set.size(), maxItems, "set count"); for (int i = 100; i < 100 + maxItems; i++) {