From 391ddba7aeb9e0d2b9c55d3c60f309260424dcbb Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Wed, 12 Feb 2025 11:59:01 -0800 Subject: [PATCH] ATLAS-4963: checkstyle compliance updates - atlas-notification module (#280) --- dev-support/checkstyle-suppressions.xml | 1 + notification/pom.xml | 5 + .../java/org/apache/atlas/hook/AtlasHook.java | 411 +++++++++--------- .../apache/atlas/hook/AtlasHookException.java | 7 +- .../apache/atlas/hook/AtlasTopicCreator.java | 42 +- .../atlas/hook/FailedMessagesLogger.java | 10 +- .../atlas/kafka/AtlasKafkaConsumer.java | 77 ++-- .../apache/atlas/kafka/AtlasKafkaMessage.java | 14 +- .../atlas/kafka/EmbeddedKafkaServer.java | 32 +- .../apache/atlas/kafka/KafkaNotification.java | 297 ++++++------- .../atlas/kafka/NotificationProvider.java | 22 +- .../AbstractMessageDeserializer.java | 13 +- .../notification/AbstractNotification.java | 145 +++--- .../AbstractNotificationConsumer.java | 7 +- .../AtlasNotificationMessageDeserializer.java | 124 +++--- .../IncompatibleVersionException.java | 8 +- .../notification/MessageDeserializer.java | 8 +- .../notification/NotificationConsumer.java | 16 +- .../notification/NotificationException.java | 14 +- .../notification/NotificationInterface.java | 55 ++- .../notification/SplitMessageAggregator.java | 7 +- .../entity/EntityMessageDeserializer.java | 13 +- .../hook/HookMessageDeserializer.java | 15 +- .../notification/rest/RestNotification.java | 127 +++--- .../atlas/notification/spool/Archiver.java | 10 +- .../notification/spool/AtlasFileSpool.java | 44 +- .../notification/spool/FileOperations.java | 6 +- .../notification/spool/IndexManagement.java | 157 ++++--- .../atlas/notification/spool/Publisher.java | 79 ++-- .../spool/SpoolConfiguration.java | 24 +- .../atlas/notification/spool/SpoolUtils.java | 75 ++-- .../atlas/notification/spool/Spooler.java | 22 +- .../spool/models/IndexRecord.java | 55 ++- .../spool/models/IndexRecords.java | 12 +- .../utils/local/FileLockedReadWrite.java | 6 +- .../spool/utils/local/FileOpAppend.java | 7 +- .../spool/utils/local/FileOpDelete.java | 6 +- .../spool/utils/local/FileOpRead.java | 6 +- .../spool/utils/local/FileOpUpdate.java | 8 +- .../spool/utils/local/FileOperation.java | 41 +- .../atlas/util/CommandHandlerUtility.java | 24 +- .../org/apache/atlas/hook/AtlasHookTest.java | 83 ++-- .../atlas/hook/AtlasTopicCreatorTest.java | 53 +-- .../apache/atlas/kafka/KafkaConsumerTest.java | 148 +++---- .../kafka/KafkaNotificationMockTest.java | 137 +++--- .../atlas/kafka/KafkaNotificationTest.java | 33 +- .../AbstractNotificationConsumerTest.java | 139 +++--- .../AbstractNotificationTest.java | 32 +- .../AtlasNotificationMessageTest.java | 51 +-- .../notification/MessageVersionTest.java | 66 +-- .../notification/RestNotificationTest.java | 70 ++- .../SplitMessageAggregatorTest.java | 20 +- .../EntityNotificationDeserializerTest.java | 16 +- .../entity/EntityNotificationTest.java | 67 ++- .../HookNotificationDeserializerTest.java | 29 +- .../hook/HookNotificationTest.java | 46 +- .../spool/AtlasFileSpoolTest.java | 169 ++++--- .../atlas/notification/spool/BaseTest.java | 22 +- .../spool/IndexManagementTest.java | 98 +++-- 59 files changed, 1649 insertions(+), 1682 deletions(-) diff --git a/dev-support/checkstyle-suppressions.xml b/dev-support/checkstyle-suppressions.xml index df58ce247a..147a4da5cc 100644 --- a/dev-support/checkstyle-suppressions.xml +++ b/dev-support/checkstyle-suppressions.xml @@ -35,4 +35,5 @@ + diff --git a/notification/pom.xml b/notification/pom.xml index 6d854d3f2e..f252ac1547 100644 --- a/notification/pom.xml +++ b/notification/pom.xml @@ -31,6 +31,11 @@ Apache Atlas Notification Apache Atlas Notification + + true + false + + diff --git a/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java b/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java index 27db3b08fd..fc307a34c0 100644 --- a/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java +++ b/notification/src/main/java/org/apache/atlas/hook/AtlasHook.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,12 +26,12 @@ import org.apache.atlas.kafka.NotificationProvider; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.notification.HookNotification; +import org.apache.atlas.model.notification.MessageSource; import org.apache.atlas.notification.NotificationException; import org.apache.atlas.notification.NotificationInterface; import org.apache.atlas.utils.AtlasConfigurationUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; -import org.apache.atlas.model.notification.MessageSource; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ShutdownHookManager; @@ -50,7 +50,6 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; - /** * A base class for atlas hooks. */ @@ -74,9 +73,11 @@ public abstract class AtlasHook { public static final String ATLAS_HOOK_ENTITY_IGNORE_PATTERN = "atlas.hook.entity.ignore.pattern"; public static final String ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; - protected static Configuration atlasProperties; - protected static NotificationInterface notificationInterface; - protected MessageSource source; + public static final boolean isRESTNotificationEnabled; + public static final boolean isHookMsgsSortEnabled; + + protected static final Configuration atlasProperties; + protected static final NotificationInterface notificationInterface; private static final String metadataNamespace; private static final int SHUTDOWN_HOOK_WAIT_TIME_MS = 3000; @@ -84,161 +85,24 @@ public abstract class AtlasHook { private static final FailedMessagesLogger failedMessagesLogger; private static final int notificationMaxRetries; private static final int notificationRetryInterval; - private static ExecutorService executor = null; - public static final boolean isRESTNotificationEnabled; - public static final boolean isHookMsgsSortEnabled; - private static final List entitiesToIgnore = new ArrayList<>(); - private static boolean shouldPreprocess = false; - - - static { - try { - atlasProperties = ApplicationProperties.get(); - } catch (Exception e) { - LOG.info("Failed to load application properties", e); - } - - logFailedMessages = atlasProperties.getBoolean(ATLAS_NOTIFICATION_LOG_FAILED_MESSAGES_ENABLED_KEY, true); - - if (logFailedMessages) { - failedMessagesLogger = new FailedMessagesLogger(); - } else { - failedMessagesLogger = null; - } - - isRESTNotificationEnabled = AtlasConfiguration.NOTIFICATION_HOOK_REST_ENABLED.getBoolean(); - isHookMsgsSortEnabled = atlasProperties.getBoolean(CONF_ATLAS_HOOK_MESSAGES_SORT_ENABLED, isRESTNotificationEnabled); - metadataNamespace = getMetadataNamespace(atlasProperties); - notificationMaxRetries = atlasProperties.getInt(ATLAS_NOTIFICATION_MAX_RETRIES, 3); - notificationRetryInterval = atlasProperties.getInt(ATLAS_NOTIFICATION_RETRY_INTERVAL, 1000); - notificationInterface = NotificationProvider.get(); - - String[] patternsToIgnoreEntities = atlasProperties.getStringArray(ATLAS_HOOK_ENTITY_IGNORE_PATTERN); - - if (patternsToIgnoreEntities != null) { - for (String pattern: patternsToIgnoreEntities) { - try { - entitiesToIgnore.add(Pattern.compile(pattern)); - } catch (Throwable t) { - LOG.warn("failed to compile pattern {}", pattern, t); - LOG.warn("Ignoring invalid pattern in configuration {}: {}", ATLAS_HOOK_ENTITY_IGNORE_PATTERN, pattern); - } - } - LOG.info("{}={}", ATLAS_HOOK_ENTITY_IGNORE_PATTERN, entitiesToIgnore); - } - - - shouldPreprocess = CollectionUtils.isNotEmpty(entitiesToIgnore); + private static final List entitiesToIgnore = new ArrayList<>(); + private static final boolean shouldPreprocess; + private static final ExecutorService executor; - String currentUser = ""; - - try { - currentUser = getUser(); - } catch (Exception excp) { - LOG.warn("Error in determining current user", excp); - } - - notificationInterface.setCurrentUser(currentUser); - - boolean isAsync = atlasProperties.getBoolean(ATLAS_NOTIFICATION_ASYNCHRONOUS, Boolean.TRUE); - - if (isAsync) { - int minThreads = atlasProperties.getInt(ATLAS_NOTIFICATION_ASYNCHRONOUS_MIN_THREADS, 1); - int maxThreads = atlasProperties.getInt(ATLAS_NOTIFICATION_ASYNCHRONOUS_MAX_THREADS, 1); - long keepAliveTimeMs = atlasProperties.getLong(ATLAS_NOTIFICATION_ASYNCHRONOUS_KEEP_ALIVE_TIME_MS, 10000); - int queueSize = atlasProperties.getInt(ATLAS_NOTIFICATION_ASYNCHRONOUS_QUEUE_SIZE, 10000); - - executor = new ThreadPoolExecutor(minThreads, maxThreads, keepAliveTimeMs, TimeUnit.MILLISECONDS, - new LinkedBlockingDeque<>(queueSize), - new ThreadFactoryBuilder().setNameFormat("Atlas Notifier %d").setDaemon(true).build()); - - ShutdownHookManager.get().addShutdownHook(new Thread() { - @Override - public void run() { - try { - LOG.info("==> Shutdown of Atlas Hook"); - - notificationInterface.close(); - executor.shutdown(); - executor.awaitTermination(SHUTDOWN_HOOK_WAIT_TIME_MS, TimeUnit.MILLISECONDS); - executor = null; - } catch (InterruptedException excp) { - LOG.info("Interrupt received in shutdown.", excp); - } finally { - LOG.info("<== Shutdown of Atlas Hook"); - } - } - }, AtlasConstants.ATLAS_SHUTDOWN_HOOK_PRIORITY); - } - - LOG.info("Created Atlas Hook"); - } + protected MessageSource source; public AtlasHook() { source = new MessageSource(getMessageSource()); + notificationInterface.init(this.getClass().getSimpleName(), failedMessagesLogger); } public AtlasHook(String name) { source = new MessageSource(getMessageSource()); - LOG.info("AtlasHook: Spool name: Passed from caller.: {}", name); - notificationInterface.init(name, failedMessagesLogger); - } - - public abstract String getMessageSource(); - - protected static boolean isMatch(String qualifiedName, List patterns) { - return patterns.stream().anyMatch((Pattern pattern) -> pattern.matcher(qualifiedName).matches()); - } - - private static AtlasEntity.AtlasEntitiesWithExtInfo getAtlasEntitiesWithExtInfo(HookNotification hookNotification) { - AtlasEntity.AtlasEntitiesWithExtInfo entitiesWithExtInfo = null; - switch (hookNotification.getType()) { - case ENTITY_CREATE_V2: - entitiesWithExtInfo = ((HookNotification.EntityCreateRequestV2) hookNotification).getEntities(); - break; - case ENTITY_FULL_UPDATE_V2: - entitiesWithExtInfo = ((HookNotification.EntityUpdateRequestV2) hookNotification).getEntities(); - break; - } - return entitiesWithExtInfo; - - } - private static void preprocessEntities(List hookNotifications) { - for (int i = 0; i < hookNotifications.size(); i++) { - HookNotification hookNotification = hookNotifications.get(i); - - AtlasEntity.AtlasEntitiesWithExtInfo entitiesWithExtInfo = getAtlasEntitiesWithExtInfo(hookNotification); - - if (entitiesWithExtInfo == null) { - return; - } - - List entities = entitiesWithExtInfo.getEntities(); - entities = ((entities != null) ? entities : Collections.emptyList()); - entities.removeIf((AtlasEntity entity) -> isMatch(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME).toString(), entitiesToIgnore)); - - - Map referredEntitiesMap = entitiesWithExtInfo.getReferredEntities(); - referredEntitiesMap = ((referredEntitiesMap != null) ? referredEntitiesMap: Collections.emptyMap()); - referredEntitiesMap.entrySet().removeIf((Map.Entry entry) -> isMatch(entry.getValue().getAttribute(ATTRIBUTE_QUALIFIED_NAME).toString(), entitiesToIgnore)); - - - if (CollectionUtils.isEmpty(entities) && CollectionUtils.isEmpty(referredEntitiesMap.values())) { - hookNotifications.remove(i--); + LOG.info("AtlasHook: Spool name: Passed from caller.: {}", name); - LOG.info("ignored message: {}", hookNotification); - } - } - } - private static void notifyEntitiesPostPreprocess(List messages, UserGroupInformation ugi, int maxRetries, MessageSource source) { - if (shouldPreprocess) { - preprocessEntities(messages); - } - if (CollectionUtils.isNotEmpty(messages)) { - notifyEntitiesInternal(messages, maxRetries, ugi, notificationInterface, logFailedMessages, failedMessagesLogger, source); - } + notificationInterface.init(name, failedMessagesLogger); } /** @@ -254,19 +118,59 @@ public static void notifyEntities(List messages, UserGroupInfo if (executor == null) { // send synchronously notifyEntitiesPostPreprocess(messages, ugi, maxRetries, source); } else { - executor.submit(new Runnable() { - @Override - public void run() { - notifyEntitiesPostPreprocess(messages, ugi, maxRetries, source); - } - }); + executor.submit(() -> notifyEntitiesPostPreprocess(messages, ugi, maxRetries, source)); + } + } + + /** + * Returns the logged in user. + * + * @return + */ + public static String getUser() { + return getUser(null, null); + } + + public static String getUser(String userName) { + return getUser(userName, null); + } + + /** + * Returns the user. Order of preference: + * 1. Given userName + * 2. ugi.getShortUserName() + * 3. UserGroupInformation.getCurrentUser().getShortUserName() + * 4. System.getProperty("user.name") + */ + + public static String getUser(String userName, UserGroupInformation ugi) { + if (StringUtils.isNotEmpty(userName)) { + LOG.debug("Returning userName {}", userName); + + return userName; + } + + if (ugi != null && StringUtils.isNotEmpty(ugi.getShortUserName())) { + LOG.debug("Returning ugi.getShortUserName {}", userName); + + return ugi.getShortUserName(); + } + + try { + return UserGroupInformation.getCurrentUser().getShortUserName(); + } catch (IOException e) { + LOG.warn("Failed for UserGroupInformation.getCurrentUser() ", e); + + return System.getProperty("user.name"); } } + protected static boolean isMatch(String qualifiedName, List patterns) { + return patterns.stream().anyMatch((Pattern pattern) -> pattern.matcher(qualifiedName).matches()); + } + @VisibleForTesting - static void notifyEntitiesInternal(List messages, int maxRetries, UserGroupInformation ugi, - NotificationInterface notificationInterface, - boolean shouldLogFailedMessages, FailedMessagesLogger logger, MessageSource source) { + static void notifyEntitiesInternal(List messages, int maxRetries, UserGroupInformation ugi, NotificationInterface notificationInterface, boolean shouldLogFailedMessages, FailedMessagesLogger logger, MessageSource source) { if (messages == null || messages.isEmpty()) { return; } @@ -291,12 +195,10 @@ static void notifyEntitiesInternal(List messages, int maxRetri if (ugi == null) { notificationInterface.send(NotificationInterface.NotificationType.HOOK, messages, source); } else { - PrivilegedExceptionAction privilegedNotify = new PrivilegedExceptionAction() { - @Override - public Object run() throws Exception { - notificationInterface.send(NotificationInterface.NotificationType.HOOK, messages, source); - return messages; - } + PrivilegedExceptionAction privilegedNotify = () -> { + notificationInterface.send(NotificationInterface.NotificationType.HOOK, messages, source); + + return messages; }; ugi.doAs(privilegedNotify); @@ -321,10 +223,16 @@ public Object run() throws Exception { } } - LOG.error("Giving up after {} failed attempts to send notification to Atlas: {}", maxAttempts, messages.toString(), notificationFailure); + LOG.error("Giving up after {} failed attempts to send notification to Atlas: {}", maxAttempts, messages, notificationFailure); } } + public abstract String getMessageSource(); + + public String getMetadataNamespace() { + return metadataNamespace; + } + /** * Notify atlas of the entity through message. The entity can be a * complex entity with reference to other entities. @@ -337,47 +245,60 @@ protected void notifyEntities(List messages, UserGroupInformat notifyEntities(messages, ugi, notificationMaxRetries, source); } - /** - * Returns the logged in user. - * - * @return - */ - public static String getUser() { - return getUser(null, null); - } + private static AtlasEntity.AtlasEntitiesWithExtInfo getAtlasEntitiesWithExtInfo(HookNotification hookNotification) { + final AtlasEntity.AtlasEntitiesWithExtInfo entitiesWithExtInfo; - public static String getUser(String userName) { - return getUser(userName, null); + switch (hookNotification.getType()) { + case ENTITY_CREATE_V2: + entitiesWithExtInfo = ((HookNotification.EntityCreateRequestV2) hookNotification).getEntities(); + break; + case ENTITY_FULL_UPDATE_V2: + entitiesWithExtInfo = ((HookNotification.EntityUpdateRequestV2) hookNotification).getEntities(); + break; + default: + entitiesWithExtInfo = null; + } + + return entitiesWithExtInfo; } - /** - * Returns the user. Order of preference: - * 1. Given userName - * 2. ugi.getShortUserName() - * 3. UserGroupInformation.getCurrentUser().getShortUserName() - * 4. System.getProperty("user.name") - */ + private static void preprocessEntities(List hookNotifications) { + for (int i = 0; i < hookNotifications.size(); i++) { + HookNotification hookNotification = hookNotifications.get(i); - public static String getUser(String userName, UserGroupInformation ugi) { - if (StringUtils.isNotEmpty(userName)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Returning userName {}", userName); + AtlasEntity.AtlasEntitiesWithExtInfo entitiesWithExtInfo = getAtlasEntitiesWithExtInfo(hookNotification); + + if (entitiesWithExtInfo == null) { + return; } - return userName; - } - if (ugi != null && StringUtils.isNotEmpty(ugi.getShortUserName())) { - if (LOG.isDebugEnabled()) { - LOG.debug("Returning ugi.getShortUserName {}", userName); + List entities = entitiesWithExtInfo.getEntities(); + + entities = ((entities != null) ? entities : Collections.emptyList()); + + entities.removeIf((AtlasEntity entity) -> isMatch(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME).toString(), entitiesToIgnore)); + + Map referredEntitiesMap = entitiesWithExtInfo.getReferredEntities(); + + referredEntitiesMap = ((referredEntitiesMap != null) ? referredEntitiesMap : Collections.emptyMap()); + + referredEntitiesMap.entrySet().removeIf((Map.Entry entry) -> isMatch(entry.getValue().getAttribute(ATTRIBUTE_QUALIFIED_NAME).toString(), entitiesToIgnore)); + + if (CollectionUtils.isEmpty(entities) && CollectionUtils.isEmpty(referredEntitiesMap.values())) { + hookNotifications.remove(i--); + + LOG.info("ignored message: {}", hookNotification); } - return ugi.getShortUserName(); } + } - try { - return UserGroupInformation.getCurrentUser().getShortUserName(); - } catch (IOException e) { - LOG.warn("Failed for UserGroupInformation.getCurrentUser() ", e); - return System.getProperty("user.name"); + private static void notifyEntitiesPostPreprocess(List messages, UserGroupInformation ugi, int maxRetries, MessageSource source) { + if (shouldPreprocess) { + preprocessEntities(messages); + } + + if (CollectionUtils.isNotEmpty(messages)) { + notifyEntitiesInternal(messages, maxRetries, ugi, notificationInterface, logFailedMessages, failedMessagesLogger, source); } } @@ -389,7 +310,91 @@ private static String getClusterName(Configuration config) { return config.getString(CLUSTER_NAME_KEY, DEFAULT_CLUSTER_NAME); } - public String getMetadataNamespace() { - return metadataNamespace; + static { + Configuration conf = null; + + try { + conf = ApplicationProperties.get(); + } catch (Exception e) { + LOG.info("Failed to load application properties", e); + } + + atlasProperties = conf; + logFailedMessages = atlasProperties.getBoolean(ATLAS_NOTIFICATION_LOG_FAILED_MESSAGES_ENABLED_KEY, true); + + if (logFailedMessages) { + failedMessagesLogger = new FailedMessagesLogger(); + } else { + failedMessagesLogger = null; + } + + isRESTNotificationEnabled = AtlasConfiguration.NOTIFICATION_HOOK_REST_ENABLED.getBoolean(); + isHookMsgsSortEnabled = atlasProperties.getBoolean(CONF_ATLAS_HOOK_MESSAGES_SORT_ENABLED, isRESTNotificationEnabled); + metadataNamespace = getMetadataNamespace(atlasProperties); + notificationMaxRetries = atlasProperties.getInt(ATLAS_NOTIFICATION_MAX_RETRIES, 3); + notificationRetryInterval = atlasProperties.getInt(ATLAS_NOTIFICATION_RETRY_INTERVAL, 1000); + notificationInterface = NotificationProvider.get(); + + String[] patternsToIgnoreEntities = atlasProperties.getStringArray(ATLAS_HOOK_ENTITY_IGNORE_PATTERN); + + if (patternsToIgnoreEntities != null) { + for (String pattern : patternsToIgnoreEntities) { + try { + entitiesToIgnore.add(Pattern.compile(pattern)); + } catch (Throwable t) { + LOG.warn("failed to compile pattern {}", pattern, t); + LOG.warn("Ignoring invalid pattern in configuration {}: {}", ATLAS_HOOK_ENTITY_IGNORE_PATTERN, pattern); + } + } + + LOG.info("{}={}", ATLAS_HOOK_ENTITY_IGNORE_PATTERN, entitiesToIgnore); + } + + shouldPreprocess = CollectionUtils.isNotEmpty(entitiesToIgnore); + + String currentUser = ""; + + try { + currentUser = getUser(); + } catch (Exception excp) { + LOG.warn("Error in determining current user", excp); + } + + notificationInterface.setCurrentUser(currentUser); + + boolean isAsync = atlasProperties.getBoolean(ATLAS_NOTIFICATION_ASYNCHRONOUS, Boolean.TRUE); + + if (isAsync) { + int minThreads = atlasProperties.getInt(ATLAS_NOTIFICATION_ASYNCHRONOUS_MIN_THREADS, 1); + int maxThreads = atlasProperties.getInt(ATLAS_NOTIFICATION_ASYNCHRONOUS_MAX_THREADS, 1); + long keepAliveTimeMs = atlasProperties.getLong(ATLAS_NOTIFICATION_ASYNCHRONOUS_KEEP_ALIVE_TIME_MS, 10000); + int queueSize = atlasProperties.getInt(ATLAS_NOTIFICATION_ASYNCHRONOUS_QUEUE_SIZE, 10000); + + executor = new ThreadPoolExecutor(minThreads, maxThreads, keepAliveTimeMs, TimeUnit.MILLISECONDS, + new LinkedBlockingDeque<>(queueSize), + new ThreadFactoryBuilder().setNameFormat("Atlas Notifier %d").setDaemon(true).build()); + + ShutdownHookManager.get().addShutdownHook(new Thread() { + @Override + public void run() { + try { + LOG.info("==> Shutdown of Atlas Hook"); + + notificationInterface.close(); + executor.shutdown(); + + boolean ignored = executor.awaitTermination(SHUTDOWN_HOOK_WAIT_TIME_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException excp) { + LOG.info("Interrupt received in shutdown.", excp); + } finally { + LOG.info("<== Shutdown of Atlas Hook"); + } + } + }, AtlasConstants.ATLAS_SHUTDOWN_HOOK_PRIORITY); + } else { + executor = null; + } + + LOG.info("Created Atlas Hook"); } -} \ No newline at end of file +} diff --git a/notification/src/main/java/org/apache/atlas/hook/AtlasHookException.java b/notification/src/main/java/org/apache/atlas/hook/AtlasHookException.java index 01014ec5b2..86d66a972c 100644 --- a/notification/src/main/java/org/apache/atlas/hook/AtlasHookException.java +++ b/notification/src/main/java/org/apache/atlas/hook/AtlasHookException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,7 +21,6 @@ * Exception class for Atlas Hooks. */ public class AtlasHookException extends Exception { - public AtlasHookException() { } diff --git a/notification/src/main/java/org/apache/atlas/hook/AtlasTopicCreator.java b/notification/src/main/java/org/apache/atlas/hook/AtlasTopicCreator.java index 80a12ac33a..1400d5ca7e 100644 --- a/notification/src/main/java/org/apache/atlas/hook/AtlasTopicCreator.java +++ b/notification/src/main/java/org/apache/atlas/hook/AtlasTopicCreator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,11 +39,17 @@ * Use this class to create a Kafka topic with specific configuration like number of partitions, replicas, etc. */ public class AtlasTopicCreator { - private static final Logger LOG = LoggerFactory.getLogger(AtlasTopicCreator.class); public static final String ATLAS_NOTIFICATION_CREATE_TOPICS_KEY = "atlas.notification.create.topics"; + public static void main(String[] args) throws AtlasException { + Configuration configuration = ApplicationProperties.get(); + AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); + + atlasTopicCreator.createAtlasTopic(configuration, args); + } + /** * Create an Atlas topic. * @@ -61,35 +67,41 @@ public void createAtlasTopic(Configuration atlasProperties, String... topicNames if (!handleSecurity(atlasProperties)) { return; } - try(KafkaUtils kafkaUtils = getKafkaUtils(atlasProperties)) { + + try (KafkaUtils kafkaUtils = getKafkaUtils(atlasProperties)) { int numPartitions = atlasProperties.getInt("atlas.notification.partitions", 1); - int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1); + int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1); + kafkaUtils.createTopics(Arrays.asList(topicNames), numPartitions, numReplicas); } catch (Exception e) { - LOG.error("Error while creating topics e :" + e.getMessage(), e); + LOG.error("Error while creating topics e :{}", e.getMessage(), e); } } else { - LOG.info("Not creating topics {} as {} is false", StringUtils.join(topicNames, ","), - ATLAS_NOTIFICATION_CREATE_TOPICS_KEY); + LOG.info("Not creating topics {} as {} is false", StringUtils.join(topicNames, ","), ATLAS_NOTIFICATION_CREATE_TOPICS_KEY); } } @VisibleForTesting protected boolean handleSecurity(Configuration atlasProperties) { if (AuthenticationUtil.isKerberosAuthenticationEnabled(atlasProperties)) { - String kafkaPrincipal = atlasProperties.getString("atlas.notification.kafka.service.principal"); - String kafkaKeyTab = atlasProperties.getString("atlas.notification.kafka.keytab.location"); - org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); + String kafkaPrincipal = atlasProperties.getString("atlas.notification.kafka.service.principal"); + String kafkaKeyTab = atlasProperties.getString("atlas.notification.kafka.keytab.location"); + org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); + SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, hadoopConf); + try { String serverPrincipal = SecurityUtil.getServerPrincipal(kafkaPrincipal, (String) null); + UserGroupInformation.setConfiguration(hadoopConf); UserGroupInformation.loginUserFromKeytab(serverPrincipal, kafkaKeyTab); } catch (IOException e) { LOG.warn("Could not login as {} from keytab file {}", kafkaPrincipal, kafkaKeyTab, e); + return false; } } + return true; } @@ -97,10 +109,4 @@ protected boolean handleSecurity(Configuration atlasProperties) { KafkaUtils getKafkaUtils(Configuration configuration) { return new KafkaUtils(configuration); } - - public static void main(String[] args) throws AtlasException { - Configuration configuration = ApplicationProperties.get(); - AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); - atlasTopicCreator.createAtlasTopic(configuration, args); - } } diff --git a/notification/src/main/java/org/apache/atlas/hook/FailedMessagesLogger.java b/notification/src/main/java/org/apache/atlas/hook/FailedMessagesLogger.java index 30b5902204..6b7ad078f0 100644 --- a/notification/src/main/java/org/apache/atlas/hook/FailedMessagesLogger.java +++ b/notification/src/main/java/org/apache/atlas/hook/FailedMessagesLogger.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,11 +18,9 @@ package org.apache.atlas.hook; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * A logger wrapper that can be used to write messages that failed to be sent to a log file. */ @@ -32,4 +30,4 @@ public class FailedMessagesLogger { public void log(String message) { LOG.info(message); } -} \ No newline at end of file +} diff --git a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java index 89ec59caa5..9c1537f2c1 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java +++ b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +21,11 @@ import org.apache.atlas.notification.AtlasNotificationMessageDeserializer; import org.apache.atlas.notification.NotificationInterface; import org.apache.commons.collections.MapUtils; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,12 +34,6 @@ import java.util.List; import java.util.Map; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; - /** * Kafka specific notification consumer. * @@ -45,7 +44,7 @@ public class AtlasKafkaConsumer extends AbstractNotificationConsumer { private final KafkaConsumer kafkaConsumer; private final boolean autoCommitEnabled; - private long pollTimeoutMilliSeconds = 1000L; + private final long pollTimeoutMilliSeconds; public AtlasKafkaConsumer(NotificationInterface.NotificationType notificationType, KafkaConsumer kafkaConsumer, boolean autoCommitEnabled, long pollTimeoutMilliSeconds) { this(notificationType.getDeserializer(), kafkaConsumer, autoCommitEnabled, pollTimeoutMilliSeconds); @@ -59,32 +58,11 @@ public AtlasKafkaConsumer(AtlasNotificationMessageDeserializer deserializer, this.pollTimeoutMilliSeconds = pollTimeoutMilliSeconds; } - public List> receive() { - return this.receive(this.pollTimeoutMilliSeconds); - } - - @Override - public List> receive(long timeoutMilliSeconds) { - return receive(this.pollTimeoutMilliSeconds, null); - } - - @Override - public List> receiveWithCheckedCommit(Map lastCommittedPartitionOffset) { - return receive(this.pollTimeoutMilliSeconds, lastCommittedPartitionOffset); - } - - @Override - public List> receiveRawRecordsWithCheckedCommit(Map lastCommittedPartitionOffset) { - return receiveRawRecords(this.pollTimeoutMilliSeconds, lastCommittedPartitionOffset); - } - - @Override public void commit(TopicPartition partition, long offset) { if (!autoCommitEnabled) { - if (LOG.isDebugEnabled()) { - LOG.info(" commiting the offset ==>> " + offset); - } + LOG.debug(" commiting the offset ==>> {}", offset); + kafkaConsumer.commitSync(Collections.singletonMap(partition, new OffsetAndMetadata(offset))); } } @@ -103,6 +81,25 @@ public void wakeup() { } } + public List> receive() { + return this.receive(this.pollTimeoutMilliSeconds); + } + + @Override + public List> receive(long timeoutMilliSeconds) { + return receive(this.pollTimeoutMilliSeconds, null); + } + + @Override + public List> receiveWithCheckedCommit(Map lastCommittedPartitionOffset) { + return receive(this.pollTimeoutMilliSeconds, lastCommittedPartitionOffset); + } + + @Override + public List> receiveRawRecordsWithCheckedCommit(Map lastCommittedPartitionOffset) { + return receiveRawRecords(this.pollTimeoutMilliSeconds, lastCommittedPartitionOffset); + } + private List> receiveRawRecords(long timeoutMilliSeconds, Map lastCommittedPartitionOffset) { return receive(timeoutMilliSeconds, lastCommittedPartitionOffset, true); } @@ -112,7 +109,7 @@ private List> receive(long timeoutMilliSeconds, Map> receive(long timeoutMilliSeconds, Map lastCommittedPartitionOffset, boolean isRawDataRequired) { - List> messages = new ArrayList(); + List> messages = new ArrayList<>(); ConsumerRecords records = kafkaConsumer != null ? kafkaConsumer.poll(timeoutMilliSeconds) : null; @@ -127,10 +124,9 @@ private List> receive(long timeoutMilliSeconds, Map> receive(long timeoutMilliSeconds, Map> receive(long timeoutMilliSeconds, Map + * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -79,7 +79,11 @@ public long getMsgCreated() { return this.msgCreated; } - public String getSource() { return this.source; } + public String getSource() { + return this.source; + } - public String getRawRecordData() { return this.rawRecordData; } + public String getRawRecordData() { + return this.rawRecordData; + } } diff --git a/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java b/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java index 0a1f02add2..ee87763528 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java +++ b/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,9 +23,10 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.service.Service; +import org.apache.atlas.util.CommandHandlerUtility; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationConverter; -import org.apache.kafka.clients.producer.*; +import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.utils.Time; import org.apache.zookeeper.server.NIOServerCnxnFactory; @@ -35,37 +36,36 @@ import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import org.apache.atlas.util.CommandHandlerUtility; import scala.Option; import javax.inject.Inject; + import java.io.File; import java.io.IOException; +import java.net.BindException; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; -import java.net.BindException; -import java.util.*; +import java.util.Properties; @Component @Order(3) public class EmbeddedKafkaServer implements Service { public static final Logger LOG = LoggerFactory.getLogger(EmbeddedKafkaServer.class); - public static final String PROPERTY_PREFIX = "atlas.kafka"; - private static final String ATLAS_KAFKA_DATA = "data"; - public static final String PROPERTY_EMBEDDED = "atlas.notification.embedded"; + public static final String PROPERTY_PREFIX = "atlas.kafka"; + public static final String PROPERTY_EMBEDDED = "atlas.notification.embedded"; + private static final String ATLAS_KAFKA_DATA = "data"; private static final int MAX_RETRY_TO_ACQUIRE_PORT = 3; private final boolean isEmbedded; - private Properties properties; + private final Properties properties; private KafkaServer kafkaServer; private ServerCnxnFactory factory; - @Inject - public EmbeddedKafkaServer(Configuration applicationProperties) throws AtlasException { + public EmbeddedKafkaServer(Configuration applicationProperties) { Configuration kafkaConf = ApplicationProperties.getSubsetConfiguration(applicationProperties, PROPERTY_PREFIX); this.isEmbedded = applicationProperties.getBoolean(PROPERTY_EMBEDDED, false); @@ -110,18 +110,18 @@ private String startZk() throws IOException, InterruptedException { LOG.info("Starting zookeeper at {}", zkValue); - URL zkAddress = getURL(zkValue); + URL zkAddress = getURL(zkValue); File snapshotDir = constructDir("zk/txn"); File logDir = constructDir("zk/snap"); for (int attemptCount = 0; attemptCount < MAX_RETRY_TO_ACQUIRE_PORT; attemptCount++) { try { - factory = NIOServerCnxnFactory.createFactory(new InetSocketAddress(zkAddress.getHost(), zkAddress.getPort()), 1024); + factory = NIOServerCnxnFactory.createFactory(new InetSocketAddress(zkAddress.getHost(), zkAddress.getPort()), 1024); break; } catch (BindException e) { LOG.warn("Attempt {}: Starting zookeeper at {} failed", attemptCount, zkValue); - if(attemptCount == MAX_RETRY_TO_ACQUIRE_PORT - 1) { + if (attemptCount == MAX_RETRY_TO_ACQUIRE_PORT - 1) { throw e; } diff --git a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java index 870d508146..61e3053e48 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java +++ b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,9 +30,9 @@ import org.apache.commons.configuration.ConfigurationConverter; import org.apache.commons.lang.StringUtils; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; @@ -42,11 +42,17 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; -import java.util.*; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; import java.util.concurrent.Future; -import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY; import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED; +import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY; import static org.apache.atlas.security.SecurityUtil.getPassword; /** @@ -57,58 +63,29 @@ public class KafkaNotification extends AbstractNotification implements Service { public static final Logger LOG = LoggerFactory.getLogger(KafkaNotification.class); - public static final String PROPERTY_PREFIX = "atlas.kafka"; - public static final String UNSORTED_POSTFIX = "_UNSORTED"; - public static final String ATLAS_HOOK_TOPIC = AtlasConfiguration.NOTIFICATION_HOOK_TOPIC_NAME.getString(); - public static final String ATLAS_ENTITIES_TOPIC = AtlasConfiguration.NOTIFICATION_ENTITIES_TOPIC_NAME.getString(); - protected static final String CONSUMER_GROUP_ID_PROPERTY = "group.id"; - - private static final String[] ATLAS_HOOK_CONSUMER_TOPICS = AtlasConfiguration.NOTIFICATION_HOOK_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_HOOK_TOPIC); - private static final String[] ATLAS_ENTITIES_CONSUMER_TOPICS = AtlasConfiguration.NOTIFICATION_ENTITIES_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_ENTITIES_TOPIC); + public static final String PROPERTY_PREFIX = "atlas.kafka"; + public static final String UNSORTED_POSTFIX = "_UNSORTED"; + public static final String ATLAS_HOOK_TOPIC = AtlasConfiguration.NOTIFICATION_HOOK_TOPIC_NAME.getString(); + public static final String ATLAS_ENTITIES_TOPIC = AtlasConfiguration.NOTIFICATION_ENTITIES_TOPIC_NAME.getString(); - private static final String DEFAULT_CONSUMER_CLOSED_ERROR_MESSAGE = "This consumer has already been closed."; + public static String ATLAS_HOOK_TOPIC_UNSORTED; + public static String[] ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS; - public static String ATLAS_HOOK_TOPIC_UNSORTED; - public static String[] ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS; - - static { - try { - ATLAS_HOOK_TOPIC_UNSORTED = ATLAS_HOOK_TOPIC + UNSORTED_POSTFIX; - ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS = ATLAS_HOOK_CONSUMER_TOPICS != null && ATLAS_HOOK_CONSUMER_TOPICS.length > 0 - ? new String[ATLAS_HOOK_CONSUMER_TOPICS.length] : new String[] {ATLAS_HOOK_TOPIC_UNSORTED}; - - for (int i = 0; i < ATLAS_HOOK_CONSUMER_TOPICS.length; i++) { - ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS[i] = ATLAS_HOOK_CONSUMER_TOPICS[i] + UNSORTED_POSTFIX; - } - } catch (Exception e) { - LOG.error("Error while initializing Kafka Notification", e); - } - } + protected static final String CONSUMER_GROUP_ID_PROPERTY = "group.id"; - private static final Map PRODUCER_TOPIC_MAP = new HashMap() { - { - put(NotificationType.HOOK, ATLAS_HOOK_TOPIC); - put(NotificationType.HOOK_UNSORTED, ATLAS_HOOK_TOPIC_UNSORTED); - put(NotificationType.ENTITIES, ATLAS_ENTITIES_TOPIC); - } - }; + private static final String[] ATLAS_HOOK_CONSUMER_TOPICS = AtlasConfiguration.NOTIFICATION_HOOK_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_HOOK_TOPIC); + private static final String[] ATLAS_ENTITIES_CONSUMER_TOPICS = AtlasConfiguration.NOTIFICATION_ENTITIES_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_ENTITIES_TOPIC); + private static final String DEFAULT_CONSUMER_CLOSED_ERROR_MESSAGE = "This consumer has already been closed."; - private static final Map CONSUMER_TOPICS_MAP = new HashMap() { - { - put(NotificationType.HOOK, trimAndPurge(ATLAS_HOOK_CONSUMER_TOPICS)); - put(NotificationType.HOOK_UNSORTED, trimAndPurge(ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS)); - put(NotificationType.ENTITIES, trimAndPurge(ATLAS_ENTITIES_CONSUMER_TOPICS)); - } - }; + private static final Map PRODUCER_TOPIC_MAP = new HashMap<>(); + private static final Map CONSUMER_TOPICS_MAP = new HashMap<>(); private final Properties properties; private final Long pollTimeOutMs; - private final Map> consumers = new HashMap<>(); - private final Map producers = new HashMap<>(); - private String consumerClosedErrorMsg; + private final Map> consumers = new HashMap<>(); + private final Map producers = new HashMap<>(); private final Map producersByTopic = new HashMap<>(); - - // ----- Constructors ---------------------------------------------------- + private String consumerClosedErrorMsg; /** * Construct a KafkaNotification. @@ -142,7 +119,7 @@ public KafkaNotification(Configuration applicationProperties) throws AtlasExcept properties.put("enable.auto.commit", kafkaConf.getBoolean("enable.auto.commit", oldApiCommitEnableFlag)); properties.put("session.timeout.ms", kafkaConf.getString("session.timeout.ms", "30000")); - if(applicationProperties.getBoolean(TLS_ENABLED, false)) { + if (applicationProperties.getBoolean(TLS_ENABLED, false)) { try { properties.put("ssl.truststore.password", getPassword(applicationProperties, TRUSTSTORE_PASSWORD_KEY)); } catch (Exception e) { @@ -158,6 +135,8 @@ public KafkaNotification(Configuration applicationProperties) throws AtlasExcept LOG.info("<== KafkaNotification()"); } + // ----- Constructors ---------------------------------------------------- + @VisibleForTesting protected KafkaNotification(Properties properties) { super(); @@ -170,12 +149,21 @@ protected KafkaNotification(Properties properties) { LOG.info("<== KafkaNotification()"); } - @VisibleForTesting - String getProducerTopicName(NotificationType notificationType) { - return PRODUCER_TOPIC_MAP.get(notificationType); - } + public static String[] trimAndPurge(String[] strings) { + List ret = new ArrayList<>(); - // ----- Service --------------------------------------------------------- + if (strings != null) { + for (String string : strings) { + String str = StringUtils.trim(string); + + if (StringUtils.isNotEmpty(str)) { + ret.add(str); + } + } + } + + return ret.toArray(new String[ret.size()]); + } @Override public void start() throws AtlasException { @@ -184,6 +172,8 @@ public void start() throws AtlasException { LOG.info("<== KafkaNotification.start()"); } + // ----- Service --------------------------------------------------------- + @Override public void stop() { LOG.info("==> KafkaNotification.stop()"); @@ -191,25 +181,45 @@ public void stop() { LOG.info("<== KafkaNotification.stop()"); } + @Override + public List> createConsumers(NotificationType notificationType, int numConsumers) { + return createConsumers(notificationType, numConsumers, Boolean.parseBoolean(properties.getProperty("enable.auto.commit", properties.getProperty("auto.commit.enable", "false")))); + } + + @Override + public void close() { + LOG.info("==> KafkaNotification.close()"); + + for (KafkaProducer producer : producers.values()) { + if (producer != null) { + try { + producer.close(); + } catch (Throwable t) { + LOG.error("failed to close Kafka producer. Ignoring", t); + } + } + } + + producers.clear(); + + LOG.info("<== KafkaNotification.close()"); + } // ----- NotificationInterface ------------------------------------------- public boolean isReady(NotificationType notificationType) { try { KafkaProducer producer = getOrCreateProducer(notificationType); + producer.metrics(); + return true; - } - catch (Exception exception) { + } catch (Exception exception) { LOG.error("Error: Connecting... {}", exception.getMessage()); + return false; } } - @Override - public List> createConsumers(NotificationType notificationType, int numConsumers) { - return createConsumers(notificationType, numConsumers, Boolean.valueOf(properties.getProperty("enable.auto.commit", properties.getProperty("auto.commit.enable","false")))); - } - @VisibleForTesting public List> createConsumers(NotificationType notificationType, int numConsumers, boolean autoCommitEnabled) { LOG.info("==> KafkaNotification.createConsumers(notificationType={}, numConsumers={}, autoCommitEnabled={})", notificationType, numConsumers, autoCommitEnabled); @@ -249,7 +259,7 @@ public List> createConsumers(NotificationType notifi notificationConsumers.add(kafkaConsumer); } - consumers.add(new AtlasKafkaConsumer(notificationType, kafkaConsumer, autoCommitEnabled, pollTimeOutMs)); + consumers.add(new AtlasKafkaConsumer<>(notificationType, kafkaConsumer, autoCommitEnabled, pollTimeOutMs)); } LOG.info("<== KafkaNotification.createConsumers(notificationType={}, numConsumers={}, autoCommitEnabled={})", notificationType, numConsumers, autoCommitEnabled); @@ -257,32 +267,16 @@ public List> createConsumers(NotificationType notifi return consumers; } - @Override - public void close() { - LOG.info("==> KafkaNotification.close()"); - - for (KafkaProducer producer : producers.values()) { - if (producer != null) { - try { - producer.close(); - } catch (Throwable t) { - LOG.error("failed to close Kafka producer. Ignoring", t); - } - } - } - - producers.clear(); - - LOG.info("<== KafkaNotification.close()"); - } - //Sending messages received through HTTP or REST Notification Service to Producer public void sendInternal(String topic, List messages, boolean isSortNeeded) throws NotificationException { KafkaProducer producer; + if (isSortNeeded) { topic = topic + UNSORTED_POSTFIX; } + producer = getOrCreateProducer(topic); + sendInternalToProducer(producer, topic, messages); } @@ -300,54 +294,12 @@ public void sendInternal(NotificationType notificationType, List message sendInternalToProducer(producer, notificationType, messages); } - @VisibleForTesting - void sendInternalToProducer(Producer p, NotificationType notificationType, List messages) throws NotificationException { - String topic = PRODUCER_TOPIC_MAP.get(notificationType); - sendInternalToProducer(p, topic, messages); - } - - void sendInternalToProducer(Producer p, String topic , List messages) throws NotificationException { - List messageContexts = new ArrayList<>(); - - for (String message : messages) { - ProducerRecord record = new ProducerRecord(topic, message); - - if (LOG.isDebugEnabled()) { - LOG.debug("Sending message for topic {}: {}", topic, message); - } - - Future future = p.send(record); - - messageContexts.add(new MessageContext(future, message)); - } - - List failedMessages = new ArrayList<>(); - Exception lastFailureException = null; - - for (MessageContext context : messageContexts) { - try { - RecordMetadata response = context.getFuture().get(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Sent message for topic - {}, partition - {}, offset - {}", response.topic(), response.partition(), response.offset()); - } - } catch (Exception e) { - lastFailureException = e; - - failedMessages.add(context.getMessage()); - } - } - - if (lastFailureException != null) { - throw new NotificationException(lastFailureException, failedMessages); - } - } - // Get properties for consumer request @VisibleForTesting public Properties getConsumerProperties(NotificationType notificationType) { // find the configured group id for the given notification type String groupId = properties.getProperty(notificationType.toString().toLowerCase() + "." + CONSUMER_GROUP_ID_PROPERTY); + if (StringUtils.isEmpty(groupId)) { groupId = "atlas"; } @@ -377,7 +329,7 @@ public KafkaConsumer getOrCreateKafkaConsumer(KafkaConsumer existingConsumer, Pr ret = new KafkaConsumer(consumerProperties); - ret.subscribe(Arrays.asList(topic)); + ret.subscribe(Collections.singletonList(topic)); } } catch (Exception ee) { LOG.error("Exception in getKafkaConsumer ", ee); @@ -386,6 +338,51 @@ public KafkaConsumer getOrCreateKafkaConsumer(KafkaConsumer existingConsumer, Pr return ret; } + @VisibleForTesting + String getProducerTopicName(NotificationType notificationType) { + return PRODUCER_TOPIC_MAP.get(notificationType); + } + + @VisibleForTesting + void sendInternalToProducer(Producer p, NotificationType notificationType, List messages) throws NotificationException { + String topic = PRODUCER_TOPIC_MAP.get(notificationType); + + sendInternalToProducer(p, topic, messages); + } + + void sendInternalToProducer(Producer p, String topic, List messages) throws NotificationException { + List messageContexts = new ArrayList<>(); + + for (String message : messages) { + ProducerRecord record = new ProducerRecord<>(topic, message); + + LOG.debug("Sending message for topic {}: {}", topic, message); + + Future future = p.send(record); + + messageContexts.add(new MessageContext(future, message)); + } + + List failedMessages = new ArrayList<>(); + Exception lastFailureException = null; + + for (MessageContext context : messageContexts) { + try { + RecordMetadata response = context.getFuture().get(); + + LOG.debug("Sent message for topic - {}, partition - {}, offset - {}", response.topic(), response.partition(), response.offset()); + } catch (Exception e) { + lastFailureException = e; + + failedMessages.add(context.getMessage()); + } + } + + if (lastFailureException != null) { + throw new NotificationException(lastFailureException, failedMessages); + } + } + private KafkaProducer getOrCreateProducer(NotificationType notificationType) { LOG.debug("==> KafkaNotification.getOrCreateProducer()"); @@ -432,23 +429,22 @@ private KafkaProducer getOrCreateProducerByCriteria(Object producerCriteria, Map return ret; } - public static String[] trimAndPurge(String[] strings) { - List ret = new ArrayList<>(); - - if (strings != null) { - for (int i = 0; i < strings.length; i++) { - String str = StringUtils.trim(strings[i]); + // kafka-client doesn't have method to check if consumer is open, hence checking list topics and catching exception + private boolean isKafkaConsumerOpen(KafkaConsumer consumer) { + boolean ret = true; - if (StringUtils.isNotEmpty(str)) { - ret.add(str); - } + try { + consumer.listTopics(); + } catch (IllegalStateException ex) { + if (ex.getMessage().equalsIgnoreCase(consumerClosedErrorMsg)) { + ret = false; } } - return ret.toArray(new String[ret.size()]); + return ret; } - private class MessageContext { + private static class MessageContext { private final Future future; private final String message; @@ -466,19 +462,24 @@ public String getMessage() { } } - // kafka-client doesn't have method to check if consumer is open, hence checking list topics and catching exception - private boolean isKafkaConsumerOpen(KafkaConsumer consumer) { - boolean ret = true; - + static { try { - consumer.listTopics(); - } catch (IllegalStateException ex) { - if (ex.getMessage().equalsIgnoreCase(consumerClosedErrorMsg)) { - ret = false; + ATLAS_HOOK_TOPIC_UNSORTED = ATLAS_HOOK_TOPIC + UNSORTED_POSTFIX; + ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS = ATLAS_HOOK_CONSUMER_TOPICS != null && ATLAS_HOOK_CONSUMER_TOPICS.length > 0 ? new String[ATLAS_HOOK_CONSUMER_TOPICS.length] : new String[] {ATLAS_HOOK_TOPIC_UNSORTED}; + + for (int i = 0; i < ATLAS_HOOK_CONSUMER_TOPICS.length; i++) { + ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS[i] = ATLAS_HOOK_CONSUMER_TOPICS[i] + UNSORTED_POSTFIX; } + } catch (Exception e) { + LOG.error("Error while initializing Kafka Notification", e); } - return ret; - } + PRODUCER_TOPIC_MAP.put(NotificationType.HOOK, ATLAS_HOOK_TOPIC); + PRODUCER_TOPIC_MAP.put(NotificationType.HOOK_UNSORTED, ATLAS_HOOK_TOPIC_UNSORTED); + PRODUCER_TOPIC_MAP.put(NotificationType.ENTITIES, ATLAS_ENTITIES_TOPIC); + CONSUMER_TOPICS_MAP.put(NotificationType.HOOK, trimAndPurge(ATLAS_HOOK_CONSUMER_TOPICS)); + CONSUMER_TOPICS_MAP.put(NotificationType.HOOK_UNSORTED, trimAndPurge(ATLAS_HOOK_UNSORTED_CONSUMER_TOPICS)); + CONSUMER_TOPICS_MAP.put(NotificationType.ENTITIES, trimAndPurge(ATLAS_ENTITIES_CONSUMER_TOPICS)); + } } diff --git a/notification/src/main/java/org/apache/atlas/kafka/NotificationProvider.java b/notification/src/main/java/org/apache/atlas/kafka/NotificationProvider.java index 9d86862576..50f6581803 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/NotificationProvider.java +++ b/notification/src/main/java/org/apache/atlas/kafka/NotificationProvider.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,19 +37,23 @@ public class NotificationProvider { private static final Logger LOG = LoggerFactory.getLogger(NotificationProvider.class); @VisibleForTesting - public static final String CONF_ATLAS_HOOK_SPOOL_ENABLED = "atlas.hook.spool.enabled"; - private static final String CONF_ATLAS_HOOK_SPOOL_DIR = "atlas.hook.spool.dir"; + public static final String CONF_ATLAS_HOOK_SPOOL_ENABLED = "atlas.hook.spool.enabled"; + private static final String CONF_ATLAS_HOOK_SPOOL_DIR = "atlas.hook.spool.dir"; private static final boolean CONF_ATLAS_HOOK_SPOOL_ENABLED_DEFAULT = false; private static NotificationInterface notificationProvider; + private NotificationProvider() { + // to block instantiation + } + public static NotificationInterface get() { if (notificationProvider == null) { try { - Configuration conf = ApplicationProperties.get(); - String spoolDir = getSpoolDir(conf); - AbstractNotification absNotifier = null; + Configuration conf = ApplicationProperties.get(); + String spoolDir = getSpoolDir(conf); + AbstractNotification absNotifier; if (AtlasHook.isRESTNotificationEnabled) { absNotifier = new RestNotification(conf); @@ -72,7 +76,9 @@ public static NotificationInterface get() { throw new RuntimeException("Error while initializing Notification interface", e); } } + LOG.debug("NotificationInterface of type {} is enabled", notificationProvider.getClass().getSimpleName()); + return notificationProvider; } diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java b/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java index fa7f488df5..91cc18aa11 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractMessageDeserializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,21 +27,16 @@ * Base notification message deserializer. */ public abstract class AbstractMessageDeserializer extends AtlasNotificationMessageDeserializer { - // ----- Constructors ---------------------------------------------------- - /** * Create a deserializer. * * @param expectedVersion the expected message version * @param notificationLogger logger for message version mismatch */ - public AbstractMessageDeserializer(TypeReference messageType, - TypeReference> notificationMessageType, - MessageVersion expectedVersion, Logger notificationLogger) { + public AbstractMessageDeserializer(TypeReference messageType, TypeReference> notificationMessageType, MessageVersion expectedVersion, Logger notificationLogger) { super(messageType, notificationMessageType, expectedVersion, notificationLogger); } - // ----- helper methods -------------------------------------------------- } diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java index cca4cb81d8..fab7fc0801 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,12 +20,12 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.AtlasException; import org.apache.atlas.model.notification.AtlasNotificationBaseMessage; +import org.apache.atlas.model.notification.AtlasNotificationBaseMessage.CompressionKind; import org.apache.atlas.model.notification.AtlasNotificationMessage; import org.apache.atlas.model.notification.AtlasNotificationStringMessage; -import org.apache.atlas.model.notification.AtlasNotificationBaseMessage.CompressionKind; import org.apache.atlas.model.notification.MessageSource; -import org.apache.atlas.type.AtlasType; import org.apache.atlas.model.notification.MessageVersion; +import org.apache.atlas.type.AtlasType; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -48,15 +48,14 @@ public abstract class AbstractNotification implements NotificationInterface { private static final Logger LOG = LoggerFactory.getLogger(AbstractNotification.class); - private static String msgIdPrefix = UUID.randomUUID().toString(); - private static AtomicInteger msgIdSuffix = new AtomicInteger(0); - /** * The current expected version for notification messages. */ public static final MessageVersion CURRENT_MESSAGE_VERSION = new MessageVersion("1.0.0"); + public static final int MAX_BYTES_PER_CHAR = 4; // each char can encode upto 4 bytes in UTF-8 - public static final int MAX_BYTES_PER_CHAR = 4; // each char can encode upto 4 bytes in UTF-8 + private static String msgIdPrefix = UUID.randomUUID().toString(); + private static AtomicInteger msgIdSuffix = new AtomicInteger(0); /** * IP address of the host in which this process has started @@ -77,75 +76,13 @@ public AbstractNotification(Configuration applicationProperties) throws AtlasExc protected AbstractNotification() { } - @Override - public void init(String source, Object failedMessagesLogger) { - } - - // ----- NotificationInterface ------------------------------------------- - - @Override - public void send(NotificationType type, List messages) throws NotificationException { - send(type, messages, new MessageSource()); - } - - @Override - public void send(NotificationType type, List messages, MessageSource source) throws NotificationException { - List strMessages = new ArrayList<>(messages.size()); - - for (int index = 0; index < messages.size(); index++) { - createNotificationMessages(messages.get(index), strMessages, source); - } - - sendInternal(type, strMessages); - } - - @Override - public void send(NotificationType type, T... messages) throws NotificationException { - send(type, Arrays.asList(messages)); - } - - @Override - public void setCurrentUser(String user) { - currentUser = user; - } - - // ----- AbstractNotification -------------------------------------------- - /** - * Send the given messages. - * - * @param type the message type - * @param messages the array of messages to send - * - * @throws NotificationException if an error occurs while sending - */ - public abstract void sendInternal(NotificationType type, List messages) throws NotificationException; - - - // ----- utility methods ------------------------------------------------- - public static String getMessageJson(Object message) { AtlasNotificationMessage notificationMsg = new AtlasNotificationMessage<>(CURRENT_MESSAGE_VERSION, message); return AtlasType.toV1Json(notificationMsg); } - private static String getHostAddress() { - if (StringUtils.isEmpty(localHostAddress)) { - try { - localHostAddress = Inet4Address.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - LOG.warn("failed to get local host address", e); - - localHostAddress = ""; - } - } - - return localHostAddress; - } - - private static String getCurrentUser() { - return currentUser; - } + // ----- NotificationInterface ------------------------------------------- /** * Get the notification message JSON from the given object. @@ -225,6 +162,68 @@ public static void createNotificationMessages(Object message, List msgJs } } + @Override + public void init(String source, Object failedMessagesLogger) { + } + + /** + * Send the given messages. + * + * @param type the message type + * @param messages the array of messages to send + * + * @throws NotificationException if an error occurs while sending + */ + public abstract void sendInternal(NotificationType type, List messages) throws NotificationException; + + private static String getHostAddress() { + if (StringUtils.isEmpty(localHostAddress)) { + try { + localHostAddress = Inet4Address.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + LOG.warn("failed to get local host address", e); + + localHostAddress = ""; + } + } + + return localHostAddress; + } + + // ----- AbstractNotification -------------------------------------------- + + private static String getCurrentUser() { + return currentUser; + } + + // ----- utility methods ------------------------------------------------- + + @Override + public void setCurrentUser(String user) { + currentUser = user; + } + + @Override + public void send(NotificationType type, T... messages) throws NotificationException { + send(type, Arrays.asList(messages)); + } + + @Override + public void send(NotificationType type, List messages) throws NotificationException { + send(type, messages, new MessageSource()); + } + + @Override + public void send(NotificationType type, List messages, MessageSource source) throws NotificationException { + List strMessages = new ArrayList<>(messages.size()); + + for (T message : messages) { + createNotificationMessages(message, strMessages, source); + } + + sendInternal(type, strMessages); + } + private static String getNextMessageId() { String nextMsgIdPrefix = msgIdPrefix; int nextMsgIdSuffix = msgIdSuffix.getAndIncrement(); @@ -234,6 +233,6 @@ private static String getNextMessageId() { msgIdSuffix = new AtomicInteger(0); } - return nextMsgIdPrefix + "_" + Integer.toString(nextMsgIdSuffix); + return nextMsgIdPrefix + "_" + nextMsgIdSuffix; } } diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java b/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java index c3940ceb44..b8d32ae37e 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractNotificationConsumer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +19,6 @@ import org.apache.kafka.common.TopicPartition; - /** * Abstract notification consumer. */ diff --git a/notification/src/main/java/org/apache/atlas/notification/AtlasNotificationMessageDeserializer.java b/notification/src/main/java/org/apache/atlas/notification/AtlasNotificationMessageDeserializer.java index 3048b9c951..6a53ffe70f 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AtlasNotificationMessageDeserializer.java +++ b/notification/src/main/java/org/apache/atlas/notification/AtlasNotificationMessageDeserializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,9 +24,9 @@ import org.apache.atlas.model.notification.AtlasNotificationBaseMessage.CompressionKind; import org.apache.atlas.model.notification.AtlasNotificationMessage; import org.apache.atlas.model.notification.AtlasNotificationStringMessage; -import org.apache.atlas.type.AtlasType; -import org.apache.atlas.model.notification.MessageVersion; import org.apache.atlas.model.notification.MessageSource; +import org.apache.atlas.model.notification.MessageVersion; +import org.apache.atlas.type.AtlasType; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,47 +47,37 @@ public abstract class AtlasNotificationMessageDeserializer implements MessageDeserializer { private static final Logger LOG = LoggerFactory.getLogger(AtlasNotificationMessageDeserializer.class); + public static final String VERSION_MISMATCH_MSG = "Notification message version mismatch. Expected %s but recieved %s. Message %s"; - public static final String VERSION_MISMATCH_MSG = - "Notification message version mismatch. Expected %s but recieved %s. Message %s"; - - private final TypeReference messageType; + private final TypeReference messageType; private final TypeReference> notificationMessageType; private final MessageVersion expectedVersion; private final Logger notificationLogger; - - private final Map splitMsgBuffer = new HashMap<>(); + private final Map splitMsgBuffer = new HashMap<>(); private final long splitMessageBufferPurgeIntervalMs; private final long splitMessageSegmentsWaitTimeMs; - private long splitMessagesLastPurgeTime = System.currentTimeMillis(); private final AtomicLong messageCountTotal = new AtomicLong(0); private final AtomicLong messageCountSinceLastInterval = new AtomicLong(0); - private long msgCreated; - private boolean spooled; - private String source; - // ----- Constructors ---------------------------------------------------- + private long splitMessagesLastPurgeTime = System.currentTimeMillis(); + private long msgCreated; + private boolean spooled; + private String source; + // ----- Constructors ---------------------------------------------------- /** * Create a notification message deserializer. * * @param expectedVersion the expected message version * @param notificationLogger logger for message version mismatch */ - public AtlasNotificationMessageDeserializer(TypeReference messageType, - TypeReference> notificationMessageType, - MessageVersion expectedVersion, Logger notificationLogger) { + public AtlasNotificationMessageDeserializer(TypeReference messageType, TypeReference> notificationMessageType, MessageVersion expectedVersion, Logger notificationLogger) { this(messageType, notificationMessageType, expectedVersion, notificationLogger, - NOTIFICATION_SPLIT_MESSAGE_SEGMENTS_WAIT_TIME_SECONDS.getLong() * 1000, - NOTIFICATION_SPLIT_MESSAGE_BUFFER_PURGE_INTERVAL_SECONDS.getLong() * 1000); + NOTIFICATION_SPLIT_MESSAGE_SEGMENTS_WAIT_TIME_SECONDS.getLong() * 1000, + NOTIFICATION_SPLIT_MESSAGE_BUFFER_PURGE_INTERVAL_SECONDS.getLong() * 1000); } - public AtlasNotificationMessageDeserializer(TypeReference messageType, - TypeReference> notificationMessageType, - MessageVersion expectedVersion, - Logger notificationLogger, - long splitMessageSegmentsWaitTimeMs, - long splitMessageBufferPurgeIntervalMs) { + public AtlasNotificationMessageDeserializer(TypeReference messageType, TypeReference> notificationMessageType, MessageVersion expectedVersion, Logger notificationLogger, long splitMessageSegmentsWaitTimeMs, long splitMessageBufferPurgeIntervalMs) { this.messageType = messageType; this.notificationMessageType = notificationMessageType; this.expectedVersion = expectedVersion; @@ -96,6 +86,37 @@ public AtlasNotificationMessageDeserializer(TypeReference messageType, this.splitMessageBufferPurgeIntervalMs = splitMessageBufferPurgeIntervalMs; } + @VisibleForTesting + static void purgeStaleMessages(Map splitMsgBuffer, long now, long maxWaitTime) { + LOG.debug("==> purgeStaleMessages(bufferedMessageCount={})", splitMsgBuffer.size()); + + List evictionList = null; + + for (SplitMessageAggregator aggregrator : splitMsgBuffer.values()) { + long waitTime = now - aggregrator.getFirstSplitTimestamp(); + + if (waitTime < maxWaitTime) { + continue; + } + + if (evictionList == null) { + evictionList = new ArrayList<>(); + } + + evictionList.add(aggregrator); + } + + if (evictionList != null) { + for (SplitMessageAggregator aggregrator : evictionList) { + LOG.error("evicting notification msgID={}, totalSplitCount={}, receivedSplitCount={}", aggregrator.getMsgId(), aggregrator.getTotalSplitCount(), aggregrator.getReceivedSplitCount()); + + splitMsgBuffer.remove(aggregrator.getMsgId()); + } + } + + LOG.debug("<== purgeStaleMessages(bufferedMessageCount={})", splitMsgBuffer.size()); + } + public TypeReference getMessageType() { return messageType; } @@ -123,18 +144,19 @@ public T deserialize(String messageJson) { messageCountTotal.incrementAndGet(); messageCountSinceLastInterval.incrementAndGet(); + this.msgCreated = 0; - this.spooled = false; - this.source = null; + this.spooled = false; + this.source = null; AtlasNotificationBaseMessage msg = AtlasType.fromV1Json(messageJson, AtlasNotificationMessage.class); if (msg == null || msg.getVersion() == null) { // older style messages not wrapped with AtlasNotificationMessage ret = AtlasType.fromV1Json(messageJson, messageType); - } else { + } else { this.msgCreated = ((AtlasNotificationMessage) msg).getMsgCreationTime(); - this.spooled = ((AtlasNotificationMessage) msg).getSpooled(); - this.source = msg.getSource() != null ? msg.getSource().getSource() : null; + this.spooled = ((AtlasNotificationMessage) msg).getSpooled(); + this.source = msg.getSource() != null ? msg.getSource().getSource() : null; String msgJson = messageJson; @@ -250,11 +272,10 @@ public T deserialize(String messageJson) { } } - long now = System.currentTimeMillis(); long timeSinceLastPurge = now - splitMessagesLastPurgeTime; - if(timeSinceLastPurge >= splitMessageBufferPurgeIntervalMs) { + if (timeSinceLastPurge >= splitMessageBufferPurgeIntervalMs) { purgeStaleMessages(splitMsgBuffer, now, splitMessageSegmentsWaitTimeMs); LOG.info("Notification processing stats: total={}, sinceLastStatsReport={}", messageCountTotal.get(), messageCountSinceLastInterval.getAndSet(0)); @@ -265,42 +286,7 @@ public T deserialize(String messageJson) { return ret; } - @VisibleForTesting - static void purgeStaleMessages(Map splitMsgBuffer, long now, long maxWaitTime) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> purgeStaleMessages(bufferedMessageCount=" + splitMsgBuffer.size() + ")"); - } - - List evictionList = null; - - for (SplitMessageAggregator aggregrator : splitMsgBuffer.values()) { - long waitTime = now - aggregrator.getFirstSplitTimestamp(); - - if (waitTime < maxWaitTime) { - continue; - } - - if(evictionList == null) { - evictionList = new ArrayList<>(); - } - - evictionList.add(aggregrator); - } - - if(evictionList != null) { - for (SplitMessageAggregator aggregrator : evictionList) { - LOG.error("evicting notification msgID={}, totalSplitCount={}, receivedSplitCount={}", aggregrator.getMsgId(), aggregrator.getTotalSplitCount(), aggregrator.getReceivedSplitCount()); - splitMsgBuffer.remove(aggregrator.getMsgId()); - } - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== purgeStaleMessages(bufferedMessageCount=" + splitMsgBuffer.size() + ")"); - } - } - // ----- helper methods -------------------------------------------------- - /** * Check the message version against the expected version. * diff --git a/notification/src/main/java/org/apache/atlas/notification/IncompatibleVersionException.java b/notification/src/main/java/org/apache/atlas/notification/IncompatibleVersionException.java index 6a5901466e..0d53956fa9 100644 --- a/notification/src/main/java/org/apache/atlas/notification/IncompatibleVersionException.java +++ b/notification/src/main/java/org/apache/atlas/notification/IncompatibleVersionException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,9 +23,7 @@ * the expected version. */ public class IncompatibleVersionException extends RuntimeException { - // ----- Constructors ---------------------------------------------------- - public IncompatibleVersionException(String message) { super(message); } diff --git a/notification/src/main/java/org/apache/atlas/notification/MessageDeserializer.java b/notification/src/main/java/org/apache/atlas/notification/MessageDeserializer.java index 7778908785..a26e529b93 100644 --- a/notification/src/main/java/org/apache/atlas/notification/MessageDeserializer.java +++ b/notification/src/main/java/org/apache/atlas/notification/MessageDeserializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,7 @@ public interface MessageDeserializer { * * @param json the JSON message * - * @return the message deserialized from the given JSON + * @return the message deserialized from the given JSON */ T deserialize(String json); } diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java b/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java index 83af92b648..7f5f5586bf 100644 --- a/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java +++ b/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,23 +17,22 @@ */ package org.apache.atlas.notification; +import org.apache.atlas.kafka.AtlasKafkaMessage; +import org.apache.kafka.common.TopicPartition; + import java.util.List; import java.util.Map; -import org.apache.kafka.common.TopicPartition; -import org.apache.atlas.kafka.AtlasKafkaMessage; - /** * Atlas notification consumer. This consumer blocks until a notification can be read. * * @param the class type of notifications returned by this consumer */ public interface NotificationConsumer { - /** * Commit the offset of messages that have been successfully processed. * - * This API should be called when messages read with {@link #next()} have been successfully processed and + * This API should be called when messages read with next() have been successfully processed and * the consumer is ready to handle the next message, which could happen even after a normal or an abnormal * restart. */ @@ -56,7 +55,6 @@ public interface NotificationConsumer { */ List> receive(long timeoutMilliSeconds); - /** * Fetch data for the topics from Kafka, if lastCommittedOffset same as message * received offset, it will proceed with commit. diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationException.java b/notification/src/main/java/org/apache/atlas/notification/NotificationException.java index 353d650225..56a8b34ed2 100644 --- a/notification/src/main/java/org/apache/atlas/notification/NotificationException.java +++ b/notification/src/main/java/org/apache/atlas/notification/NotificationException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,24 +19,30 @@ import org.apache.atlas.AtlasException; +import java.util.Collections; import java.util.List; /** * Exception from notification. */ public class NotificationException extends AtlasException { - private List failedMessages; + private final List failedMessages; public NotificationException(Exception e) { super(e); + + failedMessages = Collections.emptyList(); } public NotificationException(Exception e, String errorMsg) { super(errorMsg, e); + + failedMessages = Collections.emptyList(); } public NotificationException(Exception e, List failedMessages) { super(e); + this.failedMessages = failedMessages; } diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java b/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java index 3fb616edb6..5bf5ddbfbd 100644 --- a/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java +++ b/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,36 +33,11 @@ * */ public interface NotificationInterface { - /** * Prefix for Atlas notification related configuration properties. */ String PROPERTY_PREFIX = "atlas.notification"; - /** - * Atlas notification types. - */ - enum NotificationType { - // Notifications from the Atlas integration hooks. - HOOK(new HookMessageDeserializer()), - - // Notifications from the Atlas integration hooks - unsorted. - HOOK_UNSORTED(new HookMessageDeserializer()), - - // Notifications to entity change consumers. - ENTITIES(new EntityMessageDeserializer()); - - private final AtlasNotificationMessageDeserializer deserializer; - - NotificationType(AtlasNotificationMessageDeserializer deserializer) { - this.deserializer = deserializer; - } - - public AtlasNotificationMessageDeserializer getDeserializer() { - return deserializer; - } - } - /** * * @param source: Name of the source @@ -125,4 +100,28 @@ public AtlasNotificationMessageDeserializer getDeserializer() { * */ boolean isReady(NotificationType type); + + /** + * Atlas notification types. + */ + enum NotificationType { + // Notifications from the Atlas integration hooks. + HOOK(new HookMessageDeserializer()), + + // Notifications from the Atlas integration hooks - unsorted. + HOOK_UNSORTED(new HookMessageDeserializer()), + + // Notifications to entity change consumers. + ENTITIES(new EntityMessageDeserializer()); + + private final AtlasNotificationMessageDeserializer deserializer; + + NotificationType(AtlasNotificationMessageDeserializer deserializer) { + this.deserializer = deserializer; + } + + public AtlasNotificationMessageDeserializer getDeserializer() { + return deserializer; + } + } } diff --git a/notification/src/main/java/org/apache/atlas/notification/SplitMessageAggregator.java b/notification/src/main/java/org/apache/atlas/notification/SplitMessageAggregator.java index 10df12184b..a555d37ea2 100644 --- a/notification/src/main/java/org/apache/atlas/notification/SplitMessageAggregator.java +++ b/notification/src/main/java/org/apache/atlas/notification/SplitMessageAggregator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +17,6 @@ */ package org.apache.atlas.notification; - import org.apache.atlas.model.notification.AtlasNotificationStringMessage; public class SplitMessageAggregator { diff --git a/notification/src/main/java/org/apache/atlas/notification/entity/EntityMessageDeserializer.java b/notification/src/main/java/org/apache/atlas/notification/entity/EntityMessageDeserializer.java index 4dd5ea2dda..9673cf24cf 100644 --- a/notification/src/main/java/org/apache/atlas/notification/entity/EntityMessageDeserializer.java +++ b/notification/src/main/java/org/apache/atlas/notification/entity/EntityMessageDeserializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,22 +30,17 @@ * Entity notification message deserializer. */ public class EntityMessageDeserializer extends AbstractMessageDeserializer { - /** * Logger for entity notification messages. */ private static final Logger NOTIFICATION_LOGGER = LoggerFactory.getLogger(EntityMessageDeserializer.class); - // ----- Constructors ---------------------------------------------------- - /** * Create an entity notification message deserializer. */ public EntityMessageDeserializer() { - super(new TypeReference() {}, - new TypeReference>() {}, - AbstractNotification.CURRENT_MESSAGE_VERSION, NOTIFICATION_LOGGER); + super(new TypeReference() {}, new TypeReference>() {}, AbstractNotification.CURRENT_MESSAGE_VERSION, NOTIFICATION_LOGGER); } @Override diff --git a/notification/src/main/java/org/apache/atlas/notification/hook/HookMessageDeserializer.java b/notification/src/main/java/org/apache/atlas/notification/hook/HookMessageDeserializer.java index e83824a9f9..fbe1a6b76c 100644 --- a/notification/src/main/java/org/apache/atlas/notification/hook/HookMessageDeserializer.java +++ b/notification/src/main/java/org/apache/atlas/notification/hook/HookMessageDeserializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,28 +26,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - - /** * Hook notification message deserializer. */ public class HookMessageDeserializer extends AbstractMessageDeserializer { - /** * Logger for hook notification messages. */ private static final Logger NOTIFICATION_LOGGER = LoggerFactory.getLogger(HookMessageDeserializer.class); - // ----- Constructors ---------------------------------------------------- - /** * Create a hook notification message deserializer. */ public HookMessageDeserializer() { - super(new TypeReference() {}, - new TypeReference>() {}, - AbstractNotification.CURRENT_MESSAGE_VERSION, NOTIFICATION_LOGGER); + super(new TypeReference() {}, new TypeReference>() {}, AbstractNotification.CURRENT_MESSAGE_VERSION, NOTIFICATION_LOGGER); } @Override diff --git a/notification/src/main/java/org/apache/atlas/notification/rest/RestNotification.java b/notification/src/main/java/org/apache/atlas/notification/rest/RestNotification.java index fb598f899c..5a35b037c5 100644 --- a/notification/src/main/java/org/apache/atlas/notification/rest/RestNotification.java +++ b/notification/src/main/java/org/apache/atlas/notification/rest/RestNotification.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,51 +37,84 @@ import java.util.List; import java.util.Map; - import static org.apache.atlas.kafka.KafkaNotification.ATLAS_ENTITIES_TOPIC; import static org.apache.atlas.kafka.KafkaNotification.ATLAS_HOOK_TOPIC; public class RestNotification extends AbstractNotification { + private static final Logger LOG = LoggerFactory.getLogger(RestNotification.class); - private static final Logger LOG = LoggerFactory.getLogger(RestNotification.class); - private static final int BATCH_MAX_LENGTH_BYTES = AtlasConfiguration.NOTIFICATION_REST_BODY_MAX_LENGTH_BYTES.getInt(); - private static final String ATLAS_ENDPOINT = "atlas.rest.address"; - private static final String BASIC_AUTH_USERNAME = "atlas.rest.basic.auth.username"; - private static final String BASIC_AUTH_PASSWORD = "atlas.rest.basic.auth.password"; - private static final String DEFAULT_ATLAS_URL = "http://localhost:31000/"; - - private static final Map PRODUCER_TOPIC_MAP = new HashMap() { - { - put(NotificationType.HOOK, ATLAS_HOOK_TOPIC); - put(NotificationType.ENTITIES, ATLAS_ENTITIES_TOPIC); - } - }; + private static final int BATCH_MAX_LENGTH_BYTES = AtlasConfiguration.NOTIFICATION_REST_BODY_MAX_LENGTH_BYTES.getInt(); + private static final String ATLAS_ENDPOINT = "atlas.rest.address"; + private static final String BASIC_AUTH_USERNAME = "atlas.rest.basic.auth.username"; + private static final String BASIC_AUTH_PASSWORD = "atlas.rest.basic.auth.password"; + private static final String DEFAULT_ATLAS_URL = "http://localhost:31000/"; + + private static final Map PRODUCER_TOPIC_MAP = new HashMap<>(); @VisibleForTesting public AtlasClientV2 atlasClientV2; public RestNotification(Configuration configuration) throws AtlasException { super(); + setupAtlasClientV2(configuration); } + @Override + public void sendInternal(NotificationType type, List messages) throws NotificationException { + String topic = PRODUCER_TOPIC_MAP.get(type); + List> batches = getBatches(messages); + + int batchCounter = 0; + + try { + for (List batch : batches) { + batchCounter++; + + atlasClientV2.postNotificationToTopic(topic, batch); + } + } catch (AtlasServiceException e) { + if (e.getMessage().contains(AtlasErrorCode.NOTIFICATION_EXCEPTION.getErrorCode())) { + LOG.error("Sending notifications through REST interface failed starting from batch# {}", batchCounter); + + throw new NotificationException(e); + } else { + throw new RuntimeException(e); + } + } + } + + @Override + public List> createConsumers(NotificationType notificationType, int numConsumers) { + return null; + } + + @Override + public void close() { + } + + @Override + public boolean isReady(NotificationType type) { + return true; + } + private AtlasClientV2 setupAtlasClientV2(Configuration configuration) throws AtlasException { if (atlasClientV2 != null) { return atlasClientV2; } + try { String[] atlasEndPoint = configuration.getStringArray(ATLAS_ENDPOINT); if (atlasEndPoint == null || atlasEndPoint.length == 0) { - atlasEndPoint = new String[] { DEFAULT_ATLAS_URL }; + atlasEndPoint = new String[] {DEFAULT_ATLAS_URL}; } if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) { - String fileAuthUsername = configuration.getString(BASIC_AUTH_USERNAME, "admin"); - String fileAuthPassword = configuration.getString(BASIC_AUTH_PASSWORD, "admin123"); - String[] basicAuthUsernamePassword = (fileAuthUsername == null || fileAuthPassword == null) - ? AuthenticationUtil.getBasicAuthenticationInput() - : new String[]{fileAuthUsername, fileAuthPassword}; + String fileAuthUsername = configuration.getString(BASIC_AUTH_USERNAME, "admin"); + String fileAuthPassword = configuration.getString(BASIC_AUTH_PASSWORD, "admin123"); + + String[] basicAuthUsernamePassword = (fileAuthUsername == null || fileAuthPassword == null) ? AuthenticationUtil.getBasicAuthenticationInput() : new String[] {fileAuthUsername, fileAuthPassword}; atlasClientV2 = new AtlasClientV2(atlasEndPoint, basicAuthUsernamePassword); } else { @@ -94,31 +127,10 @@ private AtlasClientV2 setupAtlasClientV2(Configuration configuration) throws Atl return atlasClientV2; } - @Override - public void sendInternal(NotificationType type, List messages) throws NotificationException { - String topic = PRODUCER_TOPIC_MAP.get(type); - List> batches = getBatches(messages); - - int batchCounter = 0; - try { - for (List batch: batches) { - batchCounter++; - atlasClientV2.postNotificationToTopic(topic, batch); - } - } catch (AtlasServiceException e) { - if (e.getMessage().contains(AtlasErrorCode.NOTIFICATION_EXCEPTION.getErrorCode())) { - LOG.error("Sending notifications through REST interface failed starting from batch# {}", batchCounter); - throw new NotificationException(e); - } else { - throw new RuntimeException(e); - } - } - } - private List> getBatches(List messages) { - List> batches = new ArrayList(); - List batch = new ArrayList(); - int batchSize = 0; + List> batches = new ArrayList<>(); + List batch = new ArrayList<>(); + int batchSize = 0; for (String message : messages) { byte[] msgBytes = AtlasNotificationBaseMessage.getBytesUtf8(message); @@ -126,28 +138,21 @@ private List> getBatches(List messages) { if (batchSize > 0 && batchSize + msgBytes.length > BATCH_MAX_LENGTH_BYTES) { batches.add(batch); - batch = new ArrayList(); + batch = new ArrayList<>(); batchSize = 0; } + batch.add(message); batchSize += msgBytes.length; } - batches.add(batch); - return batches; - } - - @Override - public List> createConsumers(NotificationType notificationType, int numConsumers) { - return null; - } - @Override - public void close() { + batches.add(batch); + return batches; } - @Override - public boolean isReady(NotificationType type) { - return true; + static { + PRODUCER_TOPIC_MAP.put(NotificationType.HOOK, ATLAS_HOOK_TOPIC); + PRODUCER_TOPIC_MAP.put(NotificationType.ENTITIES, ATLAS_ENTITIES_TOPIC); } } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/Archiver.java b/notification/src/main/java/org/apache/atlas/notification/spool/Archiver.java index 9e12d26dc2..849ae1404f 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/Archiver.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/Archiver.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,7 +52,7 @@ public void archive(IndexRecord indexRecord) { } private void moveToArchiveDir(IndexRecord indexRecord) { - File spoolFile = null; + File spoolFile = null; File archiveFile = null; try { @@ -118,7 +118,7 @@ private void removeOldFiles() { LOG.info("{}: Deleted: {} archived files", source, filesDeletedCount); } } - } catch(Exception exception){ + } catch (Exception exception) { LOG.error("{}: Error deleting older files from archive folder. Folder: {}", source, archiveFolder, exception); } } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/AtlasFileSpool.java b/notification/src/main/java/org/apache/atlas/notification/spool/AtlasFileSpool.java index 57c0c7d092..8f102e180a 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/AtlasFileSpool.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/AtlasFileSpool.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; + import static org.apache.atlas.repository.Constants.FILE_SPOOL_SOURCE; public class AtlasFileSpool implements NotificationInterface { @@ -42,7 +43,7 @@ public class AtlasFileSpool implements NotificationInterface { private final Spooler spooler; private final Publisher publisher; private Thread publisherThread; - private Boolean initDone = null; + private Boolean initDone; private String currentUser; public AtlasFileSpool(Configuration configuration, AbstractNotification notificationHandler) { @@ -89,6 +90,7 @@ public void init(String source, Object failedMessagesLogger) { @Override public void setCurrentUser(String user) { this.notificationHandler.setCurrentUser(user); + this.currentUser = user; } @@ -109,24 +111,16 @@ public void send(NotificationType type, List messages) throws Notificatio send(type, messages, new MessageSource(FILE_SPOOL_SOURCE)); } - @Override - public boolean isReady(NotificationType type) { - return true; - } - @Override public void send(NotificationType type, List messages, MessageSource source) throws NotificationException { List serializedMessages = getSerializedMessages(messages, source); + if (hasInitSucceeded() && (this.indexManagement.isPending() || this.publisher.isDestinationDown())) { - if (LOG.isDebugEnabled()) { - LOG.debug("AtlasFileSpool.send(): sending to spooler"); - } + LOG.debug("AtlasFileSpool.send(): sending to spooler"); spooler.sendInternal(type, serializedMessages); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("AtlasFileSpool.send(): sending to notificationHandler"); - } + LOG.debug("AtlasFileSpool.send(): sending to notificationHandler"); try { notificationHandler.sendInternal(type, serializedMessages); @@ -146,15 +140,6 @@ public void send(NotificationType type, List messages, MessageSource sour } } - private List getSerializedMessages(List messages, MessageSource source) { - List serializedMessages = new ArrayList<>(messages.size()); - for (int index = 0; index < messages.size(); index++) { - notificationHandler.createNotificationMessages(messages.get(index), serializedMessages, source); - } - - return serializedMessages; - } - @Override public void close() { try { @@ -168,6 +153,21 @@ public void close() { } } + @Override + public boolean isReady(NotificationType type) { + return true; + } + + private List getSerializedMessages(List messages, MessageSource source) { + List serializedMessages = new ArrayList<>(messages.size()); + + for (T message : messages) { + AbstractNotification.createNotificationMessages(message, serializedMessages, source); + } + + return serializedMessages; + } + private void startPublisher() { publisherThread = new Thread(publisher); @@ -183,6 +183,6 @@ private boolean isInitDone() { } private boolean hasInitSucceeded() { - return this.initDone != null && this.initDone == true; + return this.initDone != null && this.initDone; } } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/FileOperations.java b/notification/src/main/java/org/apache/atlas/notification/spool/FileOperations.java index 538ea49b88..52a7531439 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/FileOperations.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/FileOperations.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/IndexManagement.java b/notification/src/main/java/org/apache/atlas/notification/spool/IndexManagement.java index adbb8d1ebd..8450f4884a 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/IndexManagement.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/IndexManagement.java @@ -55,52 +55,38 @@ public void init() throws IOException, AtlasException { String sourceName = config.getSourceName(); File spoolDir = SpoolUtils.getCreateDirectoryWithPermissionCheck(config.getSpoolDir(), config.getUser()); + if (spoolDir == null) { - throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, spoolDir.getAbsolutePath())); + throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, spoolDir)); } config.setSpoolDir(spoolDir.getAbsolutePath()); File archiveDir = SpoolUtils.getCreateDirectory(config.getArchiveDir()); + if (archiveDir == null) { - throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, archiveDir.getAbsolutePath())); + throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, archiveDir)); } File indexFile = SpoolUtils.getCreateFile(config.getIndexFile(), sourceName); if (indexFile == null) { - throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, indexFile.getAbsolutePath())); + throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, indexFile)); } File indexDoneFile = SpoolUtils.getCreateFile(config.getIndexDoneFile(), sourceName); if (indexDoneFile == null) { - throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, indexDoneFile.getAbsolutePath())); + throw new AtlasException(String.format("%s: %s not found or inaccessible!", sourceName, indexDoneFile)); } performInit(indexFile.getAbsolutePath(), sourceName); } - @VisibleForTesting - void performInit(String indexFilePath, String source) { - try { - File spoolDir = config.getSpoolDir(); - File archiveDir = config.getArchiveDir(); - File indexFile = config.getIndexFile(); - File indexDoneFile = config.getIndexDoneFile(); - - indexFileManager = new IndexFileManager(source, indexFile, indexDoneFile, archiveDir, config.getMaxArchiveFiles()); - indexReader = new IndexReader(source, indexFileManager, config.getRetryDestinationMS()); - indexWriter = new IndexWriter(source, config, indexFileManager, indexReader, spoolDir, archiveDir, config.getFileRolloverSec()); - } catch (Exception e) { - LOG.error("{}: init: Failed! Error loading records from index file: {}", config.getSourceName(), indexFilePath); - } - } - public boolean isPending() { return !indexReader.isEmpty() || (indexWriter.getCurrent() != null && indexWriter.getCurrent().isStatusWriteInProgress()) - || (indexReader.currentIndexRecord != null && indexReader.currentIndexRecord.getStatus() == IndexRecord.STATUS_READ_IN_PROGRESS); + || (indexReader.currentIndexRecord != null && IndexRecord.STATUS_READ_IN_PROGRESS.equals(indexReader.currentIndexRecord.getStatus())); } public synchronized DataOutput getSpoolWriter() throws IOException { @@ -140,11 +126,6 @@ public void rolloverSpoolFileIfNeeded() { this.indexWriter.rolloverIfNeeded(); } - @VisibleForTesting - IndexFileManager getIndexFileManager() { - return this.indexFileManager; - } - public void update(IndexRecord record) { this.indexFileManager.updateIndex(record); @@ -155,6 +136,27 @@ public void flushSpoolWriter() throws IOException { this.indexWriter.flushCurrent(); } + @VisibleForTesting + void performInit(String indexFilePath, String source) { + try { + File spoolDir = config.getSpoolDir(); + File archiveDir = config.getArchiveDir(); + File indexFile = config.getIndexFile(); + File indexDoneFile = config.getIndexDoneFile(); + + indexFileManager = new IndexFileManager(source, indexFile, indexDoneFile, archiveDir, config.getMaxArchiveFiles()); + indexReader = new IndexReader(source, indexFileManager, config.getRetryDestinationMS()); + indexWriter = new IndexWriter(source, config, indexFileManager, indexReader, spoolDir, archiveDir, config.getFileRolloverSec()); + } catch (Exception e) { + LOG.error("{}: init: Failed! Error loading records from index file: {}", config.getSourceName(), indexFilePath); + } + } + + @VisibleForTesting + IndexFileManager getIndexFileManager() { + return this.indexFileManager; + } + static class IndexWriter { private final String source; private final SpoolConfiguration config; @@ -168,10 +170,7 @@ static class IndexWriter { private DataOutput currentWriter; private boolean fileWriteInProgress; - - public IndexWriter(String source, SpoolConfiguration config, IndexFileManager indexFileManager, - IndexReader indexReader, - File spoolFolder, File archiveFolder, int rollOverTimeout) { + public IndexWriter(String source, SpoolConfiguration config, IndexFileManager indexFileManager, IndexReader indexReader, File spoolFolder, File archiveFolder, int rollOverTimeout) { this.source = source; this.config = config; this.indexFileManager = indexFileManager; @@ -184,16 +183,12 @@ public IndexWriter(String source, SpoolConfiguration config, IndexFileManager in setCurrent(indexFileManager.getFirstWriteInProgressRecord()); } - public void setCurrent(IndexRecord indexRecord) { - this.currentIndexRecord = indexRecord; - } - public IndexRecord getCurrent() { return this.currentIndexRecord; } - private void setCurrentWriter(File file) throws IOException { - this.currentWriter = fileLockedReadWrite.getOutput(file); + public void setCurrent(IndexRecord indexRecord) { + this.currentIndexRecord = indexRecord; } public synchronized DataOutput getWriter() { @@ -247,19 +242,6 @@ public synchronized void rolloverIfNeeded() { } } - private boolean shouldRolloverSpoolFile() { - return currentIndexRecord != null && - (System.currentTimeMillis() - currentIndexRecord.getCreated() > this.rollOverTimeout); - } - - void flushCurrent() throws IOException { - DataOutput pw = getWriter(); - - if (pw != null) { - fileLockedReadWrite.flush(); - } - } - public void setFileWriteInProgress(boolean val) { this.fileWriteInProgress = val; } @@ -309,6 +291,22 @@ public void stop() { LOG.info("<== IndexWriter.stop(source={})", this.config.getSourceName()); } + + void flushCurrent() throws IOException { + DataOutput pw = getWriter(); + + if (pw != null) { + fileLockedReadWrite.flush(); + } + } + + private void setCurrentWriter(File file) throws IOException { + this.currentWriter = fileLockedReadWrite.getOutput(file); + } + + private boolean shouldRolloverSpoolFile() { + return currentIndexRecord != null && (System.currentTimeMillis() - currentIndexRecord.getCreated() > this.rollOverTimeout); + } } static class IndexReader { @@ -326,18 +324,8 @@ public IndexReader(String source, IndexFileManager indexFileManager, long retryD List records = indexFileManager.getRecords(); - records.stream().forEach(x -> addIfStatus(x, IndexRecord.STATUS_READ_IN_PROGRESS)); - records.stream().forEach(x -> addIfStatus(x, IndexRecord.STATUS_PENDING)); - } - - private void addIfStatus(IndexRecord record, String status) { - if (record != null && record.getStatus().equals(status)) { - if (!SpoolUtils.fileExists(record)) { - LOG.error("IndexReader.addIfStatus(source={}): file {} not found!", this.source, record.getPath()); - } else { - addToPublishQueue(record); - } - } + records.forEach(x -> addIfStatus(x, IndexRecord.STATUS_READ_IN_PROGRESS)); + records.forEach(x -> addIfStatus(x, IndexRecord.STATUS_PENDING)); } public void addToPublishQueue(IndexRecord record) { @@ -352,6 +340,7 @@ public void addToPublishQueue(IndexRecord record) { public IndexRecord next() throws InterruptedException { this.currentIndexRecord = blockingQueue.poll(retryDestinationMS, TimeUnit.MILLISECONDS); + if (this.currentIndexRecord != null) { this.currentIndexRecord.setStatus(IndexRecord.STATUS_READ_IN_PROGRESS); } @@ -380,6 +369,16 @@ public void removeAsDone(IndexRecord indexRecord) { indexFileManager.remove(indexRecord); } + + private void addIfStatus(IndexRecord record, String status) { + if (record != null && record.getStatus().equals(status)) { + if (!SpoolUtils.fileExists(record)) { + LOG.error("IndexReader.addIfStatus(source={}): file {} not found!", this.source, record.getPath()); + } else { + addToPublishQueue(record); + } + } + } } static class IndexFileManager { @@ -446,24 +445,6 @@ public void updateIndex(IndexRecord record) { fileOperations.update(indexFile, record.getId(), SpoolUtils.getRecordForWriting(record)); } - private void compactFile(File file) { - LOG.info("IndexFileManager.compactFile(source={}): compacting file {}", source, file.getAbsolutePath()); - - try { - fileOperations.compact(file); - } finally { - LOG.info("IndexFileManager.compactFile(source={}): done compacting file {}", source, file.getAbsolutePath()); - } - } - - private void appendToDoneFile(IndexRecord indexRecord) { - String json = SpoolUtils.getRecordForWriting(indexRecord); - - fileOperations.append(indexDoneFile, json); - - archiver.archive(indexRecord); - } - @VisibleForTesting IndexRecords loadRecords(File file) { String[] items = fileOperations.load(file); @@ -489,5 +470,23 @@ IndexRecord add(String path) { return record; } + + private void compactFile(File file) { + LOG.info("IndexFileManager.compactFile(source={}): compacting file {}", source, file.getAbsolutePath()); + + try { + fileOperations.compact(file); + } finally { + LOG.info("IndexFileManager.compactFile(source={}): done compacting file {}", source, file.getAbsolutePath()); + } + } + + private void appendToDoneFile(IndexRecord indexRecord) { + String json = SpoolUtils.getRecordForWriting(indexRecord); + + fileOperations.append(indexDoneFile, json); + + archiver.archive(indexRecord); + } } } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/Publisher.java b/notification/src/main/java/org/apache/atlas/notification/spool/Publisher.java index 01ead7db50..06b4bf5902 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/Publisher.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/Publisher.java @@ -29,7 +29,6 @@ import java.io.DataInput; import java.io.File; import java.io.FileNotFoundException; -import java.io.IOException; import java.nio.channels.OverlappingFileLockException; import java.util.ArrayList; import java.util.List; @@ -66,6 +65,7 @@ public void run() { while (true) { checkAndWaitIfDestinationDown(); + if (this.isDrain) { break; } @@ -75,6 +75,7 @@ public void run() { } record = fetchNext(record); + if (record != null && processAndDispatch(record)) { indexManagement.removeAsDone(record); @@ -106,31 +107,9 @@ public boolean isDestinationDown() { return isDestDown; } - private void checkAndWaitIfDestinationDown() throws InterruptedException { - isDestDown = !notificationHandler.isReady(NotificationInterface.NotificationType.HOOK); - if (isDestDown) { - LOG.info("Publisher.waitIfDestinationDown(source={}): {}: Destination is down. Sleeping for: {} ms. Queue: {} items", - this.source, notificationHandlerName, retryDestinationMS, indexManagement.getQueueSize()); - - Thread.sleep(retryDestinationMS); - } - } - - private IndexRecord fetchNext(IndexRecord record) { - if (record == null) { - try { - record = indexManagement.next(); - } catch (Exception e) { - LOG.error("Publisher.fetchNext(source={}): failed!. publisher={}", this.source, notificationHandlerName, e); - } - } - - return record; - } - @VisibleForTesting - boolean processAndDispatch(IndexRecord record) throws IOException { - boolean ret = true; + boolean processAndDispatch(IndexRecord record) { + boolean ret; if (SpoolUtils.fileExists(record)) { FileLockedReadWrite fileLockedRead = new FileLockedReadWrite(source); @@ -180,14 +159,38 @@ boolean processAndDispatch(IndexRecord record) throws IOException { return ret; } + private void checkAndWaitIfDestinationDown() throws InterruptedException { + isDestDown = !notificationHandler.isReady(NotificationInterface.NotificationType.HOOK); + + if (isDestDown) { + LOG.info("Publisher.waitIfDestinationDown(source={}): {}: Destination is down. Sleeping for: {} ms. Queue: {} items", this.source, notificationHandlerName, retryDestinationMS, indexManagement.getQueueSize()); + + Thread.sleep(retryDestinationMS); + } + } + + private IndexRecord fetchNext(IndexRecord record) { + if (record == null) { + try { + record = indexManagement.next(); + } catch (Exception e) { + LOG.error("Publisher.fetchNext(source={}): failed!. publisher={}", this.source, notificationHandlerName, e); + } + } + + return record; + } + private void dispatch(IndexRecord record, int lineInSpoolFile, List messages) throws Exception { - if (notificationHandler == null || messages == null || messages.size() == 0) { + if (notificationHandler == null || messages == null || messages.isEmpty()) { LOG.error("Publisher.dispatch(source={}): consumer={}: error sending logs", this.source, notificationHandlerName); } else { dispatch(record.getPath(), messages); record.setCurrentLine(lineInSpoolFile); + indexManagement.update(record); + isDestDown = false; } } @@ -214,24 +217,26 @@ private void dispatch(String filePath, List messages) throws Exception { /** * Reason for pauseBeforeSend: - * - EntityCorrelation is needed to be able to stitch lineage to the correct entity. - * - Background: When messages are added to Kafka queue directly, the ordering is incidentally guaranteed, where - * messages from lineage producing hooks reach immediately after messages from entities producing hooks. - * - When Spooled messages are posted onto Kafka, this order cannot be guaranteed. The entity correlation logic within Atlas - * can attach lineage to the correct entity, provided that the entity participating in the lineage is already present. - * - * This logic of entity correlation works well for majority of cases except where lineage entities are created before regular entities. - * In this case, shell entities get created in the absence of real entities. Problem is that there is 1 shell entity for any number of references. - * Circumventing this limitation is not easy. - * - * The pauseBeforeSend forces the situation where HiveMetaStore generated messages reach Kafka before lineage-producing hooks. + * - EntityCorrelation is needed to be able to stitch lineage to the correct entity. + * - Background: When messages are added to Kafka queue directly, the ordering is incidentally guaranteed, where + * messages from lineage producing hooks reach immediately after messages from entities producing hooks. + * - When Spooled messages are posted onto Kafka, this order cannot be guaranteed. The entity correlation logic within Atlas + * can attach lineage to the correct entity, provided that the entity participating in the lineage is already present. + *

+ * This logic of entity correlation works well for majority of cases except where lineage entities are created before regular entities. + * In this case, shell entities get created in the absence of real entities. Problem is that there is 1 shell entity for any number of references. + * Circumventing this limitation is not easy. + *

+ * The pauseBeforeSend forces the situation where HiveMetaStore generated messages reach Kafka before lineage-producing hooks. * * @throws InterruptedException */ private void pauseBeforeSend() throws InterruptedException { if (!configuration.isHiveMetaStore()) { int waitMs = configuration.getPauseBeforeSendSec() * 1000; + LOG.info("Waiting before dispatch: {}", waitMs); + Thread.sleep(waitMs); } } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/SpoolConfiguration.java b/notification/src/main/java/org/apache/atlas/notification/spool/SpoolConfiguration.java index 36ea7beb6f..b1bd4e6892 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/SpoolConfiguration.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/SpoolConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,15 +31,16 @@ public class SpoolConfiguration { private static final int PROP_FILE_MESSAGE_BATCH_SIZE_DEFAULT = 100; private static final String PROP_HIVE_METASTORE_NAME_DEFAULT = "HiveMetastoreHookImpl"; private static final String PROPERTY_PREFIX_SPOOL = "atlas.hook.spool."; - public static final String PROP_FILE_SPOOL_LOCAL_DIR = PROPERTY_PREFIX_SPOOL + "dir"; private static final String PROP_FILE_SPOOL_ARCHIVE_DIR = PROPERTY_PREFIX_SPOOL + "archive.dir"; private static final String PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT = PROPERTY_PREFIX_SPOOL + "archive.max.files"; - public static final String PROP_FILE_SPOOL_FILE_ROLLOVER_SEC = PROPERTY_PREFIX_SPOOL + "file.rollover.sec"; - public static final String PROP_FILE_SPOOL_DEST_RETRY_MS = PROPERTY_PREFIX_SPOOL + "destination.retry.ms"; private static final String PROP_MESSAGE_BATCH_SIZE = PROPERTY_PREFIX_SPOOL + "destination.message.batchsize"; - public static final String PROP_FILE_SPOOL_PAUSE_BEFORE_SEND_SEC = PROPERTY_PREFIX_SPOOL + "pause.before.send.sec"; private static final String PROP_HIVE_METASTORE_NAME = PROPERTY_PREFIX_SPOOL + "hivemetastore.name"; + public static final String PROP_FILE_SPOOL_LOCAL_DIR = PROPERTY_PREFIX_SPOOL + "dir"; + public static final String PROP_FILE_SPOOL_FILE_ROLLOVER_SEC = PROPERTY_PREFIX_SPOOL + "file.rollover.sec"; + public static final String PROP_FILE_SPOOL_DEST_RETRY_MS = PROPERTY_PREFIX_SPOOL + "destination.retry.ms"; + public static final String PROP_FILE_SPOOL_PAUSE_BEFORE_SEND_SEC = PROPERTY_PREFIX_SPOOL + "pause.before.send.sec"; + private final Configuration config; private final String messageHandlerName; @@ -48,15 +49,15 @@ public class SpoolConfiguration { private final int retryDestinationMS; private final int fileRollOverSec; private final int fileSpoolMaxFilesCount; - private String spoolDirPath; - private String archiveDir; private final int pauseBeforeSendSec; private final String hiveMetaStoreName; + private String spoolDirPath; + private String archiveDir; private String sourceName; - private String user; + private String user; public SpoolConfiguration(Configuration cfg, String messageHandlerName) { - this.config = cfg; + this.config = cfg; this.messageHandlerName = messageHandlerName; this.maxArchivedFilesCount = cfg.getInt(PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT, PROP_FILE_SPOOL_ARCHIVE_MAX_FILES_COUNT_DEFAULT); this.messageBatchSize = cfg.getInt(PROP_MESSAGE_BATCH_SIZE, PROP_FILE_MESSAGE_BATCH_SIZE_DEFAULT); @@ -108,6 +109,7 @@ public void setSpoolDir(String absolutePath) { public File getArchiveDir() { this.archiveDir = config.getString(PROP_FILE_SPOOL_ARCHIVE_DIR, new File(getSpoolDirPath(), PROP_FILE_SPOOL_ARCHIVE_DIR_DEFAULT).toString()); + return new File(this.archiveDir); } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/SpoolUtils.java b/notification/src/main/java/org/apache/atlas/notification/spool/SpoolUtils.java index 9ee4c80a45..d0f45b9549 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/SpoolUtils.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/SpoolUtils.java @@ -32,22 +32,25 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; -import java.text.SimpleDateFormat; public class SpoolUtils { private static final Logger LOG = LoggerFactory.getLogger(SpoolUtils.class); - private static final String USER_SPECIFIC_PATH_NAME_FORMAT = "%s-%s"; - public static final String DEFAULT_CHAR_SET = "UTF-8"; - private static final String DEFAULT_LINE_SEPARATOR = System.getProperty("line.separator"); - private static final String FILE_EXT_JSON = ".json"; - public static final String FILE_EXT_LOG = ".log"; - private static final String SPOOL_FILE_NAME_FORMAT_PREFIX = "%s.%s%s"; - private static final String INDEX_FILE_CLOSED_SUFFIX = "_closed.json"; - private static final String INDEX_FILE_PUBLISH_SUFFIX = "_publish.json"; - private static final String INDEX_FILE_NAME_FORMAT = "index-%s-%s" + FILE_EXT_JSON; - private static final String SPOOL_FILE_NAME_FORMAT = "spool-%s-%s-%s" + FILE_EXT_LOG; - private static final String RECORD_EMPTY = StringUtils.leftPad(StringUtils.EMPTY, IndexRecord.RECORD_SIZE) + SpoolUtils.getLineSeparator(); + public static final String DEFAULT_CHAR_SET = "UTF-8"; + public static final String FILE_EXT_LOG = ".log"; + private static final String USER_SPECIFIC_PATH_NAME_FORMAT = "%s-%s"; + private static final String DEFAULT_LINE_SEPARATOR = System.lineSeparator(); + private static final String FILE_EXT_JSON = ".json"; + private static final String SPOOL_FILE_NAME_FORMAT_PREFIX = "%s.%s%s"; + private static final String INDEX_FILE_CLOSED_SUFFIX = "_closed.json"; + private static final String INDEX_FILE_PUBLISH_SUFFIX = "_publish.json"; + private static final String INDEX_FILE_NAME_FORMAT = "index-%s-%s" + FILE_EXT_JSON; + private static final String SPOOL_FILE_NAME_FORMAT = "spool-%s-%s-%s" + FILE_EXT_LOG; + private static final String RECORD_EMPTY = StringUtils.leftPad(StringUtils.EMPTY, IndexRecord.RECORD_SIZE) + SpoolUtils.getLineSeparator(); + + private SpoolUtils() { + // to block instantiation + } public static File getCreateFile(File file, String source) throws IOException { if (createFileIfNotExists(file, source)) { @@ -65,8 +68,6 @@ public static boolean createFileIfNotExists(File file, String source) throws IOE if (!ret) { LOG.error("SpoolUtils.createFileIfNotExists(source={}): error creating file {}", source, file.getPath()); - - ret = false; } } @@ -76,28 +77,17 @@ public static boolean createFileIfNotExists(File file, String source) throws IOE public static File getCreateDirectoryWithPermissionCheck(File file, String user) { File ret = getCreateDirectory(file); - LOG.info("SpoolUtils.getCreateDirectory({}): Checking permissions..."); + LOG.info("SpoolUtils.getCreateDirectory({}): Checking permissions...", file); + if (!file.canWrite() || !file.canRead()) { File fileWithUserSuffix = getFileWithUserSuffix(file, user); - LOG.error("SpoolUtils.getCreateDirectory({}, {}): Insufficient permissions for user: {}! Will create: {}", - file.getAbsolutePath(), user, user, fileWithUserSuffix); - ret = getCreateDirectory(fileWithUserSuffix); - } - return ret; - } - - private static File getFileWithUserSuffix(File file, String user) { - if (!file.isDirectory()) { - return file; - } + LOG.error("SpoolUtils.getCreateDirectory({}, {}): Insufficient permissions for user: {}! Will create: {}", file.getAbsolutePath(), user, user, fileWithUserSuffix); - String absolutePath = file.getAbsolutePath(); - if (absolutePath.endsWith(File.pathSeparator)) { - absolutePath = StringUtils.removeEnd(absolutePath, File.pathSeparator); + ret = getCreateDirectory(fileWithUserSuffix); } - return new File(String.format(USER_SPECIFIC_PATH_NAME_FORMAT, absolutePath, user)); + return ret; } public static File getCreateDirectory(File file) { @@ -139,16 +129,12 @@ public static boolean fileExists(IndexRecord record) { return record != null && new File(record.getPath()).exists(); } - static String getSpoolFileName(String source, String handlerName, String guid) { - return String.format(SPOOL_FILE_NAME_FORMAT, source, handlerName, guid); - } - public static String getSpoolFilePath(SpoolConfiguration cfg, String spoolDir, String archiveFolder, String suffix) { - File ret = null; - String fileName = getSpoolFileName(cfg.getSourceName(), cfg.getMessageHandlerName(), suffix); + String fileName = getSpoolFileName(cfg.getSourceName(), cfg.getMessageHandlerName(), suffix); int lastDot = StringUtils.lastIndexOf(fileName, '.'); String baseName = fileName.substring(0, lastDot); String extension = fileName.substring(lastDot); + File ret; for (int sequence = 1; true; sequence++) { ret = new File(spoolDir, fileName); @@ -198,4 +184,21 @@ public static IndexRecords createRecords(String[] items) { return records; } + + static String getSpoolFileName(String source, String handlerName, String guid) { + return String.format(SPOOL_FILE_NAME_FORMAT, source, handlerName, guid); + } + + private static File getFileWithUserSuffix(File file, String user) { + if (!file.isDirectory()) { + return file; + } + + String absolutePath = file.getAbsolutePath(); + if (absolutePath.endsWith(File.pathSeparator)) { + absolutePath = StringUtils.removeEnd(absolutePath, File.pathSeparator); + } + + return new File(String.format(USER_SPECIFIC_PATH_NAME_FORMAT, absolutePath, user)); + } } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/Spooler.java b/notification/src/main/java/org/apache/atlas/notification/spool/Spooler.java index a918e9bb1d..df07f0aa19 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/Spooler.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/Spooler.java @@ -55,30 +55,32 @@ public List> createConsumers(org.apache.atlas.notifi return null; } + @Override + public void close() { + } + + @Override + public boolean isReady(NotificationType type) { + return true; + } + @Override public void sendInternal(NotificationType type, List messages) { for (int i = 0; i < messages.size(); i++) { AtlasNotificationMessage e = AtlasType.fromV1Json(messages.get(i), AtlasNotificationMessage.class); + e.setSpooled(true); messages.set(i, AtlasType.toV1Json(e)); } boolean ret = write(messages); + if (failedMessagesLogger != null && !ret) { writeToFailedMessages(messages); } } - @Override - public void close() { - } - - @Override - public boolean isReady(NotificationType type) { - return true; - } - @VisibleForTesting boolean write(List messages) { final boolean ret; @@ -109,7 +111,7 @@ private void writeToFailedMessages(List messages) { } private boolean writeInternal(List messages) { - boolean ret = false; + boolean ret; try { byte[] lineSeparatorBytes = SpoolUtils.getLineSeparator().getBytes(SpoolUtils.DEFAULT_CHAR_SET); diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecord.java b/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecord.java index 21dad0676a..0c8c55c596 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecord.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecord.java @@ -20,13 +20,12 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import org.apache.atlas.type.AtlasType; -import org.apache.commons.lang.StringUtils; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; + import java.io.Serializable; import java.util.UUID; @@ -34,7 +33,7 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) @XmlRootElement @XmlAccessorType(XmlAccessType.PROPERTY) @@ -85,86 +84,86 @@ public String toString() { + ", lastAttempt=" + lastAttempt + "]"; } - public void setId(String id) { - this.id = id; - } - public String getId() { return this.id; } - public void setPath(String path) { - this.path = path; + public void setId(String id) { + this.id = id; } public String getPath() { return this.path; } - public void setLine(int line) { - this.line = line; + public void setPath(String path) { + this.path = path; } public int getLine() { return line; } - public void setCreated(long fileCreateTime) { - this.created = fileCreateTime; + public void setLine(int line) { + this.line = line; } public long getCreated() { return this.created; } - public void setWriteCompleted(long writeCompleted) { - this.writeCompleted = writeCompleted; + public void setCreated(long fileCreateTime) { + this.created = fileCreateTime; } public long getWriteCompleted() { return this.writeCompleted; } - public void setDoneCompleted(long doneCompleted) { - this.doneCompleted = doneCompleted; + public void setWriteCompleted(long writeCompleted) { + this.writeCompleted = writeCompleted; } public long getDoneCompleted() { return doneCompleted; } - public void setLastSuccess(long lastSuccess) { - this.lastSuccess = lastSuccess; + public void setDoneCompleted(long doneCompleted) { + this.doneCompleted = doneCompleted; } public long getLastSuccess() { return lastSuccess; } - public void setLastFailed(long lastFailed) { - this.lastFailed = lastFailed; + public void setLastSuccess(long lastSuccess) { + this.lastSuccess = lastSuccess; } - public void setStatus(String status) { - this.status = status; + public void setLastFailed(long lastFailed) { + this.lastFailed = lastFailed; } public String getStatus() { return this.status; } - public void setLastAttempt(boolean lastAttempt) { - this.lastAttempt = lastAttempt; + public void setStatus(String status) { + this.status = status; } - public void setFailedAttempt(int failedAttempt) { - this.failedAttempt = failedAttempt; + public void setLastAttempt(boolean lastAttempt) { + this.lastAttempt = lastAttempt; } public int getFailedAttempt() { return failedAttempt; } + public void setFailedAttempt(int failedAttempt) { + this.failedAttempt = failedAttempt; + } + @JsonIgnore public void setDone() { setStatus(IndexRecord.STATUS_DONE); diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecords.java b/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecords.java index abcb83712f..74f9240663 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecords.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/models/IndexRecords.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,11 +20,12 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; + import java.io.Serializable; import java.util.LinkedHashMap; import java.util.Map; @@ -32,9 +33,8 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; - @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) @XmlRootElement @XmlAccessorType(XmlAccessType.PROPERTY) diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileLockedReadWrite.java b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileLockedReadWrite.java index 5d5ad8c761..de3dc21d13 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileLockedReadWrite.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileLockedReadWrite.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpAppend.java b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpAppend.java index bfc113dc74..8d065f173a 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpAppend.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpAppend.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,7 +23,6 @@ import java.nio.channels.FileLock; public class FileOpAppend extends FileOperation { - public FileOpAppend(String source) { super(source); } diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpDelete.java b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpDelete.java index 19243a6d97..bdd9bcb049 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpDelete.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpDelete.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpRead.java b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpRead.java index b228090a0d..3223d512a1 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpRead.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpRead.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpUpdate.java b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpUpdate.java index cd58e86c88..cd3a18fd97 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpUpdate.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOpUpdate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,8 +23,8 @@ import java.nio.channels.FileLock; public class FileOpUpdate extends FileOperation { - private String id; private final FileOpAppend fileOpAppend; + private String id; public FileOpUpdate(String source, FileOpAppend fileOpAppend) { super(source); diff --git a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOperation.java b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOperation.java index e5bf9d2dc9..c515b4e212 100644 --- a/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOperation.java +++ b/notification/src/main/java/org/apache/atlas/notification/spool/utils/local/FileOperation.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,6 +44,14 @@ public abstract class FileOperation { private File file; private String id; + public FileOperation(String source) { + this(source, false); + } + + public FileOperation(String source, boolean notifyConcurrency) { + this.source = source; + } + public static RandomAccessFile createRandomAccessFileForRead(File file) throws FileNotFoundException { return new RandomAccessFile(file, RANDOM_ACCESS_FILE_OPEN_MODE_R); } @@ -68,22 +76,10 @@ public static long find(RandomAccessFile raf, String id) throws IOException { return -1; } - public FileOperation(String source) { - this(source, false); - } - - public FileOperation(String source, boolean notifyConcurrency) { - this.source = source; - } - public String getSource() { return source; } - public void setId(String id) { - this.id = id; - } - public void perform(File file) { perform(file, StringUtils.EMPTY); } @@ -96,20 +92,28 @@ public void perform(File file, String json) { public void perform(File file, String id, String json) { this.setId(id); + perform(file, json); } public abstract FileLock run(RandomAccessFile randomAccessFile, FileChannel channel, String json) throws IOException; - protected File getFile() { return this.file; } + private void setFile(File file) { + this.file = file; + } + protected String getId() { return this.id; } + public void setId(String id) { + this.id = id; + } + protected void close(RandomAccessFile randomAccessFile, FileChannel channel, FileLock lock) { try { if (channel != null) { @@ -132,11 +136,6 @@ protected void close(RandomAccessFile randomAccessFile, FileChannel channel, Fil } } - - private void setFile(File file) { - this.file = file; - } - private void performWithRetry(File file, String json) { for (int i = 0; i < MAX_RETRY_ATTEMPTS; i++) { try { diff --git a/notification/src/main/java/org/apache/atlas/util/CommandHandlerUtility.java b/notification/src/main/java/org/apache/atlas/util/CommandHandlerUtility.java index 65d70da372..e5a96039d2 100644 --- a/notification/src/main/java/org/apache/atlas/util/CommandHandlerUtility.java +++ b/notification/src/main/java/org/apache/atlas/util/CommandHandlerUtility.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,16 +33,21 @@ public class CommandHandlerUtility { private static final String SHELL_CMD = "/bin/sh"; private static final String SHELL_CMD_OPTION = "-c"; private static final String FIND_PROCESS_ID_CMD_FORMAT = "lsof -i:%s | tail -n 1 | tr -s ' ' | cut -d' ' -f2"; - private static final String KILL_PROCESS_CMD_FORMAT = "kill %s %s" ; + private static final String KILL_PROCESS_CMD_FORMAT = "kill %s %s"; private static final int SLEEP_AFTER_SOFT_KILL_IN_MS = 4000; + private CommandHandlerUtility() { + // to block instantiation + } + public static void tryKillingProcessUsingPort(int port, boolean forceKill) { String processID = findProcessIdUsingPort(port); + sendKillToPID(processID, forceKill); } private static String findProcessIdUsingPort(int port) { - String retPID = ""; + String retPID = ""; final String[] cmd = { SHELL_CMD, @@ -52,12 +57,12 @@ private static String findProcessIdUsingPort(int port) { try { Process p = Runtime.getRuntime().exec(cmd); - retPID = new BufferedReader(new InputStreamReader(p.getInputStream())) - .lines().collect(Collectors.joining("\n")); + + retPID = new BufferedReader(new InputStreamReader(p.getInputStream())).lines().collect(Collectors.joining("\n")); if (StringUtils.isEmpty(retPID)) { - String errorMsg = new BufferedReader(new InputStreamReader(p.getErrorStream())) - .lines().collect(Collectors.joining("\n")); + String errorMsg = new BufferedReader(new InputStreamReader(p.getErrorStream())).lines().collect(Collectors.joining("\n")); + throw new IOException(errorMsg); } } catch (IOException e) { @@ -79,6 +84,7 @@ private static void sendKillToPID(String pid, boolean forceKill) { if (!forceKill) { LOG.info("Sleeping for {} milliseconds after soft kill", SLEEP_AFTER_SOFT_KILL_IN_MS); + Thread.sleep(SLEEP_AFTER_SOFT_KILL_IN_MS); } } catch (IOException | InterruptedException e) { diff --git a/notification/src/test/java/org/apache/atlas/hook/AtlasHookTest.java b/notification/src/test/java/org/apache/atlas/hook/AtlasHookTest.java index b094247917..2e9acdd65f 100644 --- a/notification/src/test/java/org/apache/atlas/hook/AtlasHookTest.java +++ b/notification/src/test/java/org/apache/atlas/hook/AtlasHookTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,8 +18,8 @@ package org.apache.atlas.hook; -import org.apache.atlas.model.notification.MessageSource; import org.apache.atlas.model.notification.HookNotification; +import org.apache.atlas.model.notification.MessageSource; import org.apache.atlas.notification.NotificationException; import org.apache.atlas.notification.NotificationInterface; import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityCreateRequest; @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.mockito.Mockito.doThrow; @@ -37,9 +38,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; - public class AtlasHookTest { - @Mock private NotificationInterface notificationInterface; @@ -51,77 +50,68 @@ public void setup() { MockitoAnnotations.initMocks(this); } - @Test (timeOut = 10000) + @Test(timeOut = 10000) public void testNotifyEntitiesDoesNotHangOnException() throws Exception { - MessageSource source = new MessageSource(this.getClass().getSimpleName()); + MessageSource source = new MessageSource(this.getClass().getSimpleName()); List hookNotifications = new ArrayList<>(); + doThrow(new NotificationException(new Exception())).when(notificationInterface) .send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); - AtlasHook.notifyEntitiesInternal(hookNotifications, 0, null, notificationInterface, false, - failedMessagesLogger, source); + + AtlasHook.notifyEntitiesInternal(hookNotifications, 0, null, notificationInterface, false, failedMessagesLogger, source); // if we've reached here, the method finished OK. } @Test public void testNotifyEntitiesRetriesOnException() throws NotificationException { - MessageSource source = new MessageSource(this.getClass().getSimpleName()); - List hookNotifications = - new ArrayList() {{ - add(new EntityCreateRequest("user")); - } - }; - doThrow(new NotificationException(new Exception())).when(notificationInterface) - .send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); - AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, false, - failedMessagesLogger, source); + MessageSource source = new MessageSource(this.getClass().getSimpleName()); + List hookNotifications = new ArrayList<>(Collections.singletonList(new EntityCreateRequest("user"))); + + doThrow(new NotificationException(new Exception())).when(notificationInterface).send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); - verify(notificationInterface, times(2)). - send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); + AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, false, failedMessagesLogger, source); + + verify(notificationInterface, times(2)).send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); } @Test public void testFailedMessageIsLoggedIfRequired() throws NotificationException { - MessageSource source = new MessageSource(this.getClass().getSimpleName()); - List hookNotifications = - new ArrayList() {{ - add(new EntityCreateRequest("user")); - } - }; - doThrow(new NotificationException(new Exception(), Arrays.asList("test message"))) + MessageSource source = new MessageSource(this.getClass().getSimpleName()); + List hookNotifications = new ArrayList<>(Collections.singletonList(new EntityCreateRequest("user"))); + + doThrow(new NotificationException(new Exception(), Collections.singletonList("test message"))) .when(notificationInterface) .send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); - AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, true, - failedMessagesLogger, source); + + AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, true, failedMessagesLogger, source); verify(failedMessagesLogger, times(1)).log("test message"); } @Test public void testFailedMessageIsNotLoggedIfNotRequired() throws NotificationException { - MessageSource source = new MessageSource(this.getClass().getSimpleName()); + MessageSource source = new MessageSource(this.getClass().getSimpleName()); List hookNotifications = new ArrayList<>(); - doThrow(new NotificationException(new Exception(), Arrays.asList("test message"))) + + doThrow(new NotificationException(new Exception(), Collections.singletonList("test message"))) .when(notificationInterface) .send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); - AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, false, - failedMessagesLogger, source); + + AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, false, failedMessagesLogger, source); verifyZeroInteractions(failedMessagesLogger); } @Test public void testAllFailedMessagesAreLogged() throws NotificationException { - MessageSource source = new MessageSource(this.getClass().getSimpleName()); - List hookNotifications = - new ArrayList() {{ - add(new EntityCreateRequest("user")); - } - }; + MessageSource source = new MessageSource(this.getClass().getSimpleName()); + List hookNotifications = new ArrayList<>(Collections.singletonList(new EntityCreateRequest("user"))); + doThrow(new NotificationException(new Exception(), Arrays.asList("test message1", "test message2"))) .when(notificationInterface) .send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); - AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, true, - failedMessagesLogger, source); + + AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, true, failedMessagesLogger, source); verify(failedMessagesLogger, times(1)).log("test message1"); verify(failedMessagesLogger, times(1)).log("test message2"); @@ -129,12 +119,13 @@ public void testAllFailedMessagesAreLogged() throws NotificationException { @Test public void testFailedMessageIsNotLoggedIfNotANotificationException() throws Exception { - MessageSource source = new MessageSource(this.getClass().getSimpleName()); + MessageSource source = new MessageSource(this.getClass().getSimpleName()); List hookNotifications = new ArrayList<>(); + doThrow(new RuntimeException("test message")).when(notificationInterface) .send(NotificationInterface.NotificationType.HOOK, hookNotifications, source); - AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, true, - failedMessagesLogger, source); + + AtlasHook.notifyEntitiesInternal(hookNotifications, 2, null, notificationInterface, true, failedMessagesLogger, source); verifyZeroInteractions(failedMessagesLogger); } diff --git a/notification/src/test/java/org/apache/atlas/hook/AtlasTopicCreatorTest.java b/notification/src/test/java/org/apache/atlas/hook/AtlasTopicCreatorTest.java index 6d1a5b6ace..b6acf9b382 100644 --- a/notification/src/test/java/org/apache/atlas/hook/AtlasTopicCreatorTest.java +++ b/notification/src/test/java/org/apache/atlas/hook/AtlasTopicCreatorTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,6 @@ import org.apache.atlas.utils.KafkaUtils; import org.apache.commons.configuration.Configuration; import org.mockito.Mockito; -import org.testng.Assert; import org.testng.annotations.Test; import java.util.concurrent.ExecutionException; @@ -33,24 +32,24 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.testng.Assert.fail; public class AtlasTopicCreatorTest { - - private final String ATLAS_HOOK_TOPIC = AtlasConfiguration.NOTIFICATION_HOOK_TOPIC_NAME.getString(); - private final String ATLAS_ENTITIES_TOPIC = AtlasConfiguration.NOTIFICATION_ENTITIES_TOPIC_NAME.getString(); + private static final String ATLAS_HOOK_TOPIC = AtlasConfiguration.NOTIFICATION_HOOK_TOPIC_NAME.getString(); + private static final String ATLAS_ENTITIES_TOPIC = AtlasConfiguration.NOTIFICATION_ENTITIES_TOPIC_NAME.getString(); @Test public void shouldNotCreateAtlasTopicIfNotConfiguredToDoSo() { - Configuration configuration = mock(Configuration.class); - when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)). - thenReturn(false); - AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); + when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(false); + + AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); AtlasTopicCreator spyAtlasTopicCreator = Mockito.spy(atlasTopicCreator); + spyAtlasTopicCreator.createAtlasTopic(configuration, ATLAS_HOOK_TOPIC); - Mockito.verify(spyAtlasTopicCreator, times(0)).handleSecurity(configuration); + Mockito.verify(spyAtlasTopicCreator, times(0)).handleSecurity(configuration); } @Test @@ -58,12 +57,13 @@ public void shouldCreateTopicIfConfiguredToDoSo() { Configuration configuration = mock(Configuration.class); KafkaUtils mockKafkaUtils = Mockito.mock(KafkaUtils.class); - when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)). - thenReturn(true); + + when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true); when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false"); - AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); + AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); AtlasTopicCreator spyAtlasTopicCreator = Mockito.spy(atlasTopicCreator); + Mockito.doReturn(mockKafkaUtils).when(spyAtlasTopicCreator).getKafkaUtils(configuration); spyAtlasTopicCreator.createAtlasTopic(configuration, ATLAS_HOOK_TOPIC); @@ -71,39 +71,40 @@ public void shouldCreateTopicIfConfiguredToDoSo() { try { verify(mockKafkaUtils).createTopics(anyList(), anyInt(), anyInt()); } catch (ExecutionException | InterruptedException e) { - Assert.fail("Caught exception while verifying createTopics: " + e.getMessage()); + fail("Caught exception while verifying createTopics: " + e.getMessage()); } - } @Test public void shouldCloseResources() { Configuration configuration = mock(Configuration.class); - when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)). - thenReturn(true); + + when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true); + when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false"); - KafkaUtils mockKafkaUtils = Mockito.mock(KafkaUtils.class); + KafkaUtils mockKafkaUtils = Mockito.mock(KafkaUtils.class); AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); AtlasTopicCreator spyAtlasTopicCreator = Mockito.spy(atlasTopicCreator); + Mockito.doReturn(mockKafkaUtils).when(spyAtlasTopicCreator).getKafkaUtils(configuration); spyAtlasTopicCreator.createAtlasTopic(configuration, ATLAS_HOOK_TOPIC); verify(mockKafkaUtils).close(); - } @Test public void shouldNotProcessTopicCreationIfSecurityFails() { Configuration configuration = mock(Configuration.class); - when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)). - thenReturn(true); - KafkaUtils mockKafkaUtils = Mockito.mock(KafkaUtils.class); - AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); + when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true); + + KafkaUtils mockKafkaUtils = Mockito.mock(KafkaUtils.class); + AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator(); AtlasTopicCreator spyAtlasTopicCreator = Mockito.spy(atlasTopicCreator); + Mockito.doReturn(mockKafkaUtils).when(spyAtlasTopicCreator).getKafkaUtils(configuration); Mockito.doReturn(false).when(spyAtlasTopicCreator).handleSecurity(configuration); @@ -112,7 +113,7 @@ public void shouldNotProcessTopicCreationIfSecurityFails() { try { verify(mockKafkaUtils, times(0)).createTopics(anyList(), anyInt(), anyInt()); } catch (ExecutionException | InterruptedException e) { - Assert.fail("Caught exception while verifying createTopics: " + e.getMessage()); + fail("Caught exception while verifying createTopics: " + e.getMessage()); } } } diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java index 425c8941b3..23c2b424fa 100644 --- a/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java +++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,17 +19,17 @@ package org.apache.atlas.kafka; import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.model.notification.AtlasNotificationMessage; import org.apache.atlas.model.notification.HookNotification; import org.apache.atlas.model.notification.MessageSource; -import org.apache.atlas.v1.model.instance.Referenceable; -import org.apache.atlas.v1.model.instance.Struct; +import org.apache.atlas.model.notification.MessageVersion; import org.apache.atlas.notification.IncompatibleVersionException; import org.apache.atlas.notification.NotificationInterface.NotificationType; -import org.apache.atlas.model.notification.AtlasNotificationMessage; import org.apache.atlas.notification.entity.EntityNotificationTest; -import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityUpdateRequest; import org.apache.atlas.type.AtlasType; -import org.apache.atlas.model.notification.MessageVersion; +import org.apache.atlas.v1.model.instance.Referenceable; +import org.apache.atlas.v1.model.instance.Struct; +import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityUpdateRequest; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -42,12 +42,14 @@ import java.util.Collections; import java.util.List; -import java.util.Map; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.testng.Assert.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; /** * KafkaConsumer tests. @@ -55,12 +57,11 @@ public class KafkaConsumerTest { private static final String TRAIT_NAME = "MyTrait"; - private static final String ATLAS_HOOK_TOPIC = AtlasConfiguration.NOTIFICATION_HOOK_TOPIC_NAME.getString(); + private static final String ATLAS_HOOK_TOPIC = AtlasConfiguration.NOTIFICATION_HOOK_TOPIC_NAME.getString(); private static final String[] ATLAS_HOOK_CONSUMER_TOPICS = KafkaNotification.trimAndPurge(AtlasConfiguration.NOTIFICATION_HOOK_CONSUMER_TOPIC_NAMES.getStringArray(ATLAS_HOOK_TOPIC)); - @Mock - private KafkaConsumer kafkaConsumer; + private KafkaConsumer kafkaConsumer; @Mock private MessageSource messageSource; @@ -73,11 +74,12 @@ public void setup() { @Test public void testReceive() throws Exception { for (String topic : ATLAS_HOOK_CONSUMER_TOPICS) { - String traitName = TRAIT_NAME + "_" + topic; - Referenceable entity = getEntity(traitName); - EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); + String traitName = TRAIT_NAME + "_" + topic; + Referenceable entity = getEntity(traitName); + EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); List> messageList = testReceiveHelper(message, topic); - assertTrue(messageList.size() > 0); + + assertFalse(messageList.isEmpty()); HookNotification consumedMessage = messageList.get(0).getMessage(); @@ -85,58 +87,38 @@ public void testReceive() throws Exception { } } - - private List> testReceiveHelper(EntityUpdateRequest message, String topic) throws Exception { - - String json = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), message)); - TopicPartition tp = new TopicPartition(topic, 0); - List> klist = Collections.singletonList(new ConsumerRecord<>(topic, 0, 0L, "mykey", json)); - Map mp = Collections.singletonMap(tp, klist); - ConsumerRecords records = new ConsumerRecords(mp); - - when(kafkaConsumer.poll(100)).thenReturn(records); - - kafkaConsumer.assign(Collections.singletonList(tp)); - - AtlasKafkaConsumer consumer = new AtlasKafkaConsumer(NotificationType.HOOK, kafkaConsumer, false, 100L); - List> messageList = consumer.receive(); - return messageList; - } - @Test - public void testNextVersionMismatch() throws Exception { + public void testNextVersionMismatch() { Referenceable entity = getEntity(TRAIT_NAME); EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); String json = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("2.0.0"), message)); - TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC,0); + TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC, 0); List> klist = Collections.singletonList(new ConsumerRecord<>(ATLAS_HOOK_TOPIC, 0, 0L, "mykey", json)); - Map mp = Collections.singletonMap(tp,klist); - ConsumerRecords records = new ConsumerRecords(mp); + ConsumerRecords records = new ConsumerRecords<>(Collections.singletonMap(tp, klist)); kafkaConsumer.assign(Collections.singletonList(tp)); when(kafkaConsumer.poll(100L)).thenReturn(records); - AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationType.HOOK, kafkaConsumer ,false, 100L); + AtlasKafkaConsumer consumer = new AtlasKafkaConsumer<>(NotificationType.HOOK, kafkaConsumer, false, 100L); try { List> messageList = consumer.receive(); - assertTrue(messageList.size() > 0); + assertFalse(messageList.isEmpty()); - HookNotification consumedMessage = messageList.get(0).getMessage(); + HookNotification ignored = messageList.get(0).getMessage(); fail("Expected VersionMismatchException!"); } catch (IncompatibleVersionException e) { e.printStackTrace(); } - } - + } @Test public void testCommitIsCalledIfAutoCommitDisabled() { - TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC,0); - AtlasKafkaConsumer consumer = new AtlasKafkaConsumer(NotificationType.HOOK, kafkaConsumer, false, 100L); + TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC, 0); + AtlasKafkaConsumer consumer = new AtlasKafkaConsumer<>(NotificationType.HOOK, kafkaConsumer, false, 100L); consumer.commit(tp, 1); @@ -145,21 +127,59 @@ public void testCommitIsCalledIfAutoCommitDisabled() { @Test public void testCommitIsNotCalledIfAutoCommitEnabled() { - TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC,0); - AtlasKafkaConsumer consumer = new AtlasKafkaConsumer(NotificationType.HOOK, kafkaConsumer, true , 100L); + TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC, 0); + AtlasKafkaConsumer consumer = new AtlasKafkaConsumer<>(NotificationType.HOOK, kafkaConsumer, true, 100L); consumer.commit(tp, 1); verify(kafkaConsumer, never()).commitSync(Collections.singletonMap(tp, new OffsetAndMetadata(1))); } + @Test + public void checkCrossCombatMessageVersionTest() { + Referenceable entity = getEntity(TRAIT_NAME); + EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); + + when(messageSource.getVersion()).thenReturn("9.9.9"); + + String json = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("2.0.0"), message, "", "", false, messageSource)); + TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC, 0); + List> klist = Collections.singletonList(new ConsumerRecord<>(ATLAS_HOOK_TOPIC, 0, 0L, "mykey", json)); + ConsumerRecords records = new ConsumerRecords<>(Collections.singletonMap(tp, klist)); + + kafkaConsumer.assign(Collections.singletonList(tp)); + + when(kafkaConsumer.poll(100L)).thenReturn(records); + + AtlasKafkaConsumer consumer = new AtlasKafkaConsumer<>(NotificationType.HOOK, kafkaConsumer, false, 100L); + + try { + List> ignored = consumer.receive(); + } catch (IncompatibleVersionException e) { + e.printStackTrace(); + } + } + + private List> testReceiveHelper(EntityUpdateRequest message, String topic) { + String json = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), message)); + TopicPartition tp = new TopicPartition(topic, 0); + List> klist = Collections.singletonList(new ConsumerRecord<>(topic, 0, 0L, "mykey", json)); + ConsumerRecords records = new ConsumerRecords<>(Collections.singletonMap(tp, klist)); + + when(kafkaConsumer.poll(100)).thenReturn(records); + + kafkaConsumer.assign(Collections.singletonList(tp)); + + AtlasKafkaConsumer consumer = new AtlasKafkaConsumer<>(NotificationType.HOOK, kafkaConsumer, false, 100L); + + return consumer.receive(); + } + private Referenceable getEntity(String traitName) { - return EntityNotificationTest.getEntity("id", new Struct(traitName, Collections.emptyMap())); + return EntityNotificationTest.getEntity("id", new Struct(traitName, Collections.emptyMap())); } - private void assertMessagesEqual(EntityUpdateRequest message, - HookNotification consumedMessage, - Referenceable entity) { + private void assertMessagesEqual(EntityUpdateRequest message, HookNotification consumedMessage, Referenceable entity) { assertEquals(consumedMessage.getType(), message.getType()); assertEquals(consumedMessage.getUser(), message.getUser()); @@ -174,28 +194,4 @@ private void assertMessagesEqual(EntityUpdateRequest message, assertEquals(deserializedEntity.getTraits(), entity.getTraits()); assertEquals(deserializedEntity.getTrait(TRAIT_NAME), entity.getTrait(TRAIT_NAME)); } - - @Test - public void checkCrossCombatMessageVersionTest() throws Exception { - Referenceable entity = getEntity(TRAIT_NAME); - EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); - when(messageSource.getVersion()).thenReturn("9.9.9"); - String json = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("2.0.0"), message,"","",false,messageSource)); - TopicPartition tp = new TopicPartition(ATLAS_HOOK_TOPIC,0); - List> klist = Collections.singletonList(new ConsumerRecord<>(ATLAS_HOOK_TOPIC, 0, 0L, "mykey", json)); - Map mp = Collections.singletonMap(tp,klist); - ConsumerRecords records = new ConsumerRecords(mp); - - kafkaConsumer.assign(Collections.singletonList(tp)); - - when(kafkaConsumer.poll(100L)).thenReturn(records); - - AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationType.HOOK, kafkaConsumer ,false, 100L); - - try { - List> messageList = consumer.receive(); - } catch (IncompatibleVersionException e) { - e.printStackTrace(); - } - } } diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java index 24b6aa9c29..919419a173 100644 --- a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java +++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,22 +17,18 @@ */ package org.apache.atlas.kafka; -import org.apache.atlas.AtlasException; import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.NotificationException; import org.apache.atlas.notification.NotificationInterface; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.TopicPartition; -import org.mockito.Mockito; import org.testng.annotations.Test; -import java.io.IOException; -import java.util.Arrays; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,75 +40,75 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; public class KafkaNotificationMockTest { - @Test - @SuppressWarnings("unchecked") - public void testCreateConsumers() throws Exception { + public void testCreateConsumers() { Properties properties = mock(Properties.class); + when(properties.getProperty("entities.group.id")).thenReturn("atlas"); + Map topicCountMap = new HashMap<>(); + topicCountMap.put(KafkaNotification.ATLAS_ENTITIES_TOPIC, 1); - final AtlasKafkaConsumer consumer1 = mock(AtlasKafkaConsumer.class); - final AtlasKafkaConsumer consumer2 = mock(AtlasKafkaConsumer.class); + final AtlasKafkaConsumer consumer1 = mock(AtlasKafkaConsumer.class); + final AtlasKafkaConsumer consumer2 = mock(AtlasKafkaConsumer.class); - KafkaNotification kafkaNotification = - new TestKafkaNotification(properties, consumer1, consumer2); + KafkaNotification kafkaNotification = new TestKafkaNotification<>(properties, consumer1, consumer2); - List> consumers = - kafkaNotification.createConsumers(NotificationInterface.NotificationType.ENTITIES, 2); + List>> consumers = kafkaNotification.createConsumers(NotificationInterface.NotificationType.ENTITIES, 2); assertEquals(consumers.size(), 2); assertTrue(consumers.contains(consumer1)); assertTrue(consumers.contains(consumer2)); } - @Test @SuppressWarnings("unchecked") - public void shouldSendMessagesSuccessfully() throws NotificationException, - ExecutionException, InterruptedException { - Properties configProperties = mock(Properties.class); + public void shouldSendMessagesSuccessfully() throws NotificationException, ExecutionException, InterruptedException { + Properties configProperties = mock(Properties.class); KafkaNotification kafkaNotification = new KafkaNotification(configProperties); - Producer producer = mock(Producer.class); - String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK); - String message = "This is a test message"; - Future returnValue = mock(Future.class); - TopicPartition topicPartition = new TopicPartition(topicName, 0); - when(returnValue.get()).thenReturn(new RecordMetadata(topicPartition, 0, 0, 0, Long.valueOf(0), 0, 0)); - ProducerRecord expectedRecord = new ProducerRecord(topicName, message); + Producer producer = mock(Producer.class); + String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK); + String message = "This is a test message"; + Future returnValue = mock(Future.class); + TopicPartition topicPartition = new TopicPartition(topicName, 0); + + when(returnValue.get()).thenReturn(new RecordMetadata(topicPartition, 0, 0, 0, 0L, 0, 0)); + + ProducerRecord expectedRecord = new ProducerRecord<>(topicName, message); + when(producer.send(expectedRecord)).thenReturn(returnValue); - kafkaNotification.sendInternalToProducer(producer, - NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[]{message})); + kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, new ArrayList<>(Collections.singletonList(message))); verify(producer).send(expectedRecord); } @Test @SuppressWarnings("unchecked") - public void shouldThrowExceptionIfProducerFails() throws NotificationException, - ExecutionException, InterruptedException { - Properties configProperties = mock(Properties.class); + public void shouldThrowExceptionIfProducerFails() throws ExecutionException, InterruptedException { + Properties configProperties = mock(Properties.class); KafkaNotification kafkaNotification = new KafkaNotification(configProperties); - Producer producer = mock(Producer.class); - String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK); - String message = "This is a test message"; - Future returnValue = mock(Future.class); + Producer producer = mock(Producer.class); + String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK); + String message = "This is a test message"; + Future returnValue = mock(Future.class); + when(returnValue.get()).thenThrow(new RuntimeException("Simulating exception")); - ProducerRecord expectedRecord = new ProducerRecord(topicName, message); + + ProducerRecord expectedRecord = new ProducerRecord<>(topicName, message); + when(producer.send(expectedRecord)).thenReturn(returnValue); try { - kafkaNotification.sendInternalToProducer(producer, - NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[]{message})); + kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, new ArrayList<>(Collections.singletonList(message))); + fail("Should have thrown NotificationException"); } catch (NotificationException e) { assertEquals(e.getFailedMessages().size(), 1); @@ -122,27 +118,33 @@ public void shouldThrowExceptionIfProducerFails() throws NotificationException, @Test @SuppressWarnings("unchecked") - public void shouldCollectAllFailedMessagesIfProducerFails() throws NotificationException, - ExecutionException, InterruptedException { - Properties configProperties = mock(Properties.class); + public void shouldCollectAllFailedMessagesIfProducerFails() throws ExecutionException, InterruptedException { + Properties configProperties = mock(Properties.class); KafkaNotification kafkaNotification = new KafkaNotification(configProperties); - Producer producer = mock(Producer.class); - String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK); - String message1 = "This is a test message1"; - String message2 = "This is a test message2"; - Future returnValue1 = mock(Future.class); + Producer producer = mock(Producer.class); + String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK); + String message1 = "This is a test message1"; + String message2 = "This is a test message2"; + Future returnValue1 = mock(Future.class); + when(returnValue1.get()).thenThrow(new RuntimeException("Simulating exception")); - Future returnValue2 = mock(Future.class); + + Future returnValue2 = mock(Future.class); + when(returnValue2.get()).thenThrow(new RuntimeException("Simulating exception")); - ProducerRecord expectedRecord1 = new ProducerRecord(topicName, message1); + + ProducerRecord expectedRecord1 = new ProducerRecord<>(topicName, message1); + when(producer.send(expectedRecord1)).thenReturn(returnValue1); - ProducerRecord expectedRecord2 = new ProducerRecord(topicName, message2); + + ProducerRecord expectedRecord2 = new ProducerRecord<>(topicName, message2); + when(producer.send(expectedRecord2)).thenReturn(returnValue1); try { - kafkaNotification.sendInternalToProducer(producer, - NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[]{message1, message2})); + kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, Arrays.asList(message1, message2)); + fail("Should have thrown NotificationException"); } catch (NotificationException e) { assertEquals(e.getFailedMessages().size(), 2); @@ -151,38 +153,35 @@ public void shouldCollectAllFailedMessagesIfProducerFails() throws NotificationE } } - class TestKafkaNotification extends KafkaNotification { - - private final AtlasKafkaConsumer consumer1; - private final AtlasKafkaConsumer consumer2; + static class TestKafkaNotification extends KafkaNotification { + private final AtlasKafkaConsumer consumer1; + private final AtlasKafkaConsumer consumer2; - TestKafkaNotification(Properties properties, - AtlasKafkaConsumer consumer1, AtlasKafkaConsumer consumer2) { + TestKafkaNotification(Properties properties, AtlasKafkaConsumer consumer1, AtlasKafkaConsumer consumer2) { super(properties); + this.consumer1 = consumer1; this.consumer2 = consumer2; } - @Override - public List> createConsumers(NotificationType notificationType, - int numConsumers) { - List consumerList = new ArrayList(); + public List> createConsumers(NotificationType notificationType, int numConsumers) { + List> consumerList = new ArrayList<>(); + consumerList.add(consumer1); consumerList.add(consumer2); + return consumerList; } - protected AtlasKafkaConsumer - createConsumers(Class type, int consumerId, boolean autoCommitEnabled) { + protected AtlasKafkaConsumer createConsumers(Class type, int consumerId, boolean autoCommitEnabled) { if (consumerId == 0) { return consumer1; } else if (consumerId == 1) { return consumer2; } + return null; } - - } } diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java index 9b3535ec94..84c14c1f23 100644 --- a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java +++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,16 +19,16 @@ package org.apache.atlas.kafka; import org.apache.atlas.ApplicationProperties; -import org.apache.atlas.v1.model.instance.Referenceable; +import org.apache.atlas.model.notification.HookNotification; import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.NotificationInterface; +import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityCreateRequest; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import org.apache.atlas.model.notification.HookNotification; import java.util.List; @@ -36,7 +36,7 @@ public class KafkaNotificationTest { private EmbeddedKafkaServer kafkaServer; - private KafkaNotification kafkaNotification; + private KafkaNotification kafkaNotification; @BeforeClass public void setup() throws Exception { @@ -44,7 +44,7 @@ public void setup() throws Exception { } @AfterClass - public void shutdown() throws Exception { + public void shutdown() { cleanUpNotificationService(); } @@ -56,22 +56,25 @@ public void testReceiveKafkaMessages() throws Exception { kafkaNotification.send(NotificationInterface.NotificationType.HOOK, new EntityCreateRequest("u4", new Referenceable("type"))); NotificationConsumer consumer = kafkaNotification.createConsumers(NotificationInterface.NotificationType.HOOK, 1).get(0); - List> messages = null ; + List> messages = null; long startTime = System.currentTimeMillis(); //fetch starting time while ((System.currentTimeMillis() - startTime) < 10000) { - messages = consumer.receive(); + messages = consumer.receive(); - if (messages.size() > 0) { + if (!messages.isEmpty()) { break; } } - int i = 1; - for (AtlasKafkaMessage msg : messages){ - HookNotification message = (HookNotification) msg.getMessage(); + if (messages != null) { + int i = 1; + + for (AtlasKafkaMessage msg : messages) { + HookNotification message = (HookNotification) msg.getMessage(); - assertEquals(message.getUser(), "u"+i++); + assertEquals(message.getUser(), "u" + i++); + } } consumer.close(); @@ -116,7 +119,7 @@ void initNotificationService() throws Exception { Thread.sleep(2000); } - void cleanUpNotificationService() throws Exception { + void cleanUpNotificationService() { if (kafkaNotification != null) { kafkaNotification.close(); kafkaNotification.stop(); diff --git a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java index aee59a3953..5bcc234717 100644 --- a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,8 +21,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import org.apache.atlas.kafka.AtlasKafkaMessage; import org.apache.atlas.model.notification.AtlasNotificationMessage; -import org.apache.atlas.type.AtlasType; import org.apache.atlas.model.notification.MessageVersion; +import org.apache.atlas.type.AtlasType; +import org.apache.kafka.common.TopicPartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.Test; @@ -37,109 +38,80 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.fail; -import org.apache.kafka.common.TopicPartition; /** * AbstractNotificationConsumer tests. */ public class AbstractNotificationConsumerTest { - @Test - public void testReceive() throws Exception { - Logger logger = mock(Logger.class); - - TestMessage testMessage1 = new TestMessage("sValue1", 99); - TestMessage testMessage2 = new TestMessage("sValue2", 98); - TestMessage testMessage3 = new TestMessage("sValue3", 97); - TestMessage testMessage4 = new TestMessage("sValue4", 96); - - List jsonList = new LinkedList<>(); + public void testReceive() { + TestMessage testMessage1 = new TestMessage("sValue1", 99); + TestMessage testMessage2 = new TestMessage("sValue2", 98); + TestMessage testMessage3 = new TestMessage("sValue3", 97); + TestMessage testMessage4 = new TestMessage("sValue4", 96); + List jsonList = new LinkedList<>(); jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage1))); jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage2))); jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage3))); jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage4))); - NotificationConsumer consumer = new TestNotificationConsumer(jsonList, logger); + NotificationConsumer consumer = new TestNotificationConsumer(jsonList, mock(Logger.class)); List> messageList = consumer.receive(); assertFalse(messageList.isEmpty()); - assertEquals(messageList.get(0).getMessage(), testMessage1); - assertEquals(messageList.get(1).getMessage(), testMessage2); - assertEquals(messageList.get(2).getMessage(), testMessage3); - assertEquals(messageList.get(3).getMessage(), testMessage4); } @Test - public void testNextBackVersion() throws Exception { - Logger logger = mock(Logger.class); + public void testNextBackVersion() { + TestMessage testMessage1 = new TestMessage("sValue1", 99); + TestMessage testMessage2 = new TestMessage("sValue2", 98); + TestMessage testMessage3 = new TestMessage("sValue3", 97); + TestMessage testMessage4 = new TestMessage("sValue4", 96); + List jsonList = new LinkedList<>(); - TestMessage testMessage1 = new TestMessage("sValue1", 99); - TestMessage testMessage2 = new TestMessage("sValue2", 98); - TestMessage testMessage3 = new TestMessage("sValue3", 97); - TestMessage testMessage4 = new TestMessage("sValue4", 96); - - List jsonList = new LinkedList<>(); - - String json1 = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage1)); - String json2 = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("0.0.5"), testMessage2)); - String json3 = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("0.5.0"), testMessage3)); - String json4 = AtlasType.toV1Json(testMessage4); - - jsonList.add(json1); - jsonList.add(json2); - jsonList.add(json3); - jsonList.add(json4); + jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage1))); + jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("0.0.5"), testMessage2))); + jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("0.5.0"), testMessage3))); + jsonList.add(AtlasType.toV1Json(testMessage4)); - NotificationConsumer consumer = new TestNotificationConsumer(jsonList, logger); + NotificationConsumer consumer = new TestNotificationConsumer(jsonList, mock(Logger.class)); List> messageList = consumer.receive(); - assertEquals(new TestMessage("sValue1", 99), messageList.get(0).getMessage()); - - assertEquals(new TestMessage("sValue2", 98), messageList.get(1).getMessage()); - - assertEquals(new TestMessage("sValue3", 97), messageList.get(2).getMessage()); - - assertEquals(new TestMessage("sValue4", 96), messageList.get(3).getMessage()); - + assertEquals(messageList.get(0).getMessage(), new TestMessage("sValue1", 99)); + assertEquals(messageList.get(1).getMessage(), new TestMessage("sValue2", 98)); + assertEquals(messageList.get(2).getMessage(), new TestMessage("sValue3", 97)); + assertEquals(messageList.get(3).getMessage(), new TestMessage("sValue4", 96)); } @Test - public void testNextForwardVersion() throws Exception { - Logger logger = mock(Logger.class); - - TestMessage testMessage1 = new TestMessage("sValue1", 99); - TestMessage testMessage2 = new TestMessage("sValue2", 98); - - List jsonList = new LinkedList<>(); + public void testNextForwardVersion() { + TestMessage testMessage1 = new TestMessage("sValue1", 99); + TestMessage testMessage2 = new TestMessage("sValue2", 98); + List jsonList = new LinkedList<>(); - String json1 = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage1)); - String json2 = AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("2.0.0"), testMessage2)); + jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("1.0.0"), testMessage1))); + jsonList.add(AtlasType.toV1Json(new AtlasNotificationMessage<>(new MessageVersion("2.0.0"), testMessage2))); - jsonList.add(json1); - jsonList.add(json2); + NotificationConsumer consumer = new TestNotificationConsumer(jsonList, mock(Logger.class)); - NotificationConsumer consumer = new TestNotificationConsumer(jsonList, logger); try { List> messageList = consumer.receive(); - messageList.get(1).getMessage(); + TestMessage ignored = messageList.get(1).getMessage(); fail("Expected VersionMismatchException!"); } catch (IncompatibleVersionException e) { - + // ignored } - } - - private static class TestMessage { private String s; private int i; @@ -169,28 +141,30 @@ public void setI(int i) { } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TestMessage that = (TestMessage) o; - return i == that.i && - Objects.equals(s, that.s); + public int hashCode() { + return Objects.hash(s, i); } @Override - public int hashCode() { - return Objects.hash(s, i); + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o == null || getClass() != o.getClass()) { + return false; + } + + TestMessage that = (TestMessage) o; + + return i == that.i && Objects.equals(s, that.s); } } private static class TestNotificationConsumer extends AbstractNotificationConsumer { private static final String TEST_TOPIC_NAME = "TEST_TOPIC"; - private final List messageList; - private int index = 0; + private final List messageList; - - public TestNotificationConsumer(List messages, Logger logger) { + public TestNotificationConsumer(List messages, Logger logger) { super(new TestMessageDeserializer()); this.messageList = messages; @@ -208,7 +182,6 @@ public void close() { @Override public void wakeup() { - } @Override @@ -218,10 +191,12 @@ public List> receive() { @Override public List> receive(long timeoutMilliSeconds) { - List> tempMessageList = new ArrayList(); - for(Object json : messageList) { - tempMessageList.add(new AtlasKafkaMessage(deserializer.deserialize((String) json), -1, TEST_TOPIC_NAME, -1)); + List> tempMessageList = new ArrayList<>(); + + for (String json : messageList) { + tempMessageList.add(new AtlasKafkaMessage<>(deserializer.deserialize(json), -1, TEST_TOPIC_NAME, -1)); } + return tempMessageList; } @@ -242,15 +217,13 @@ public static class TestMessageDeserializer extends AbstractMessageDeserializer< */ private static final Logger NOTIFICATION_LOGGER = LoggerFactory.getLogger(TestMessageDeserializer.class); - // ----- Constructors ---------------------------------------------------- /** * Create a hook notification message deserializer. */ public TestMessageDeserializer() { - super(new TypeReference() {}, new TypeReference>() {}, - AbstractNotification.CURRENT_MESSAGE_VERSION, NOTIFICATION_LOGGER); + super(new TypeReference() {}, new TypeReference>() {}, AbstractNotification.CURRENT_MESSAGE_VERSION, NOTIFICATION_LOGGER); } } } diff --git a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationTest.java b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationTest.java index 4e1c0949f5..5d7791471a 100644 --- a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,9 +19,9 @@ package org.apache.atlas.notification; import org.apache.atlas.AtlasException; -import org.apache.atlas.model.notification.MessageSource; import org.apache.atlas.model.notification.HookNotification; import org.apache.atlas.model.notification.HookNotification.HookNotificationType; +import org.apache.atlas.model.notification.MessageSource; import org.apache.atlas.notification.NotificationInterface.NotificationType; import org.apache.atlas.type.AtlasType; import org.apache.commons.configuration.Configuration; @@ -32,13 +32,12 @@ import java.util.Map; import static org.mockito.Mockito.mock; -import static org.testng.Assert.*; +import static org.testng.Assert.assertEquals; /** * AbstractNotification tests. */ public class AbstractNotificationTest { - @org.testng.annotations.Test public void testSend() throws Exception { MessageSource source = new MessageSource(); @@ -55,8 +54,8 @@ public void testSend() throws Exception { notification.send(NotificationType.HOOK, message1, message2, message3); - assertEquals(NotificationType.HOOK, notification.type); - assertEquals(3, notification.messages.size()); + assertEquals(notification.type, NotificationType.HOOK); + assertEquals(notification.messages.size(), 3); for (int i = 0; i < notification.messages.size(); i++) { assertEqualsMessageJson(notification.messages.get(i), messageJson.get(i)); @@ -88,13 +87,6 @@ public void testSend2() throws Exception { } } - public static class Test extends HookNotification { - - public Test(HookNotificationType type, String user) { - super(type, user); - } - } - // ignore msgCreationTime in Json private void assertEqualsMessageJson(String msgJsonActual, String msgJsonExpected) { Map msgActual = AtlasType.fromV1Json(msgJsonActual, Map.class); @@ -106,6 +98,12 @@ private void assertEqualsMessageJson(String msgJsonActual, String msgJsonExpecte assertEquals(msgActual, msgExpected); } + public static class Test extends HookNotification { + public Test(HookNotificationType type, String user) { + super(type, user); + } + } + public static class TestNotification extends AbstractNotification { private NotificationType type; private List messages; @@ -115,9 +113,7 @@ public TestNotification(Configuration applicationProperties) throws AtlasExcepti } @Override - public void sendInternal(NotificationType notificationType, List notificationMessages) - throws NotificationException { - + public void sendInternal(NotificationType notificationType, List notificationMessages) { type = notificationType; messages = notificationMessages; } diff --git a/notification/src/test/java/org/apache/atlas/notification/AtlasNotificationMessageTest.java b/notification/src/test/java/org/apache/atlas/notification/AtlasNotificationMessageTest.java index 463797ca09..3b97849e9a 100644 --- a/notification/src/test/java/org/apache/atlas/notification/AtlasNotificationMessageTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/AtlasNotificationMessageTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,63 +28,64 @@ import org.apache.atlas.v1.model.notification.HookNotificationV1; import org.testng.annotations.Test; +import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Collections; -import static org.testng.Assert.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; /** * AtlasNotificationMessage tests. */ public class AtlasNotificationMessageTest { - @Test - public void testGetVersion() throws Exception { - MessageVersion version = new MessageVersion("1.0.0"); + public void testGetVersion() { + MessageVersion version = new MessageVersion("1.0.0"); AtlasNotificationMessage atlasNotificationMessage = new AtlasNotificationMessage<>(version, "a"); + assertEquals(atlasNotificationMessage.getVersion(), version); } @Test - public void testGetMessage() throws Exception { - String message = "a"; - MessageVersion version = new MessageVersion("1.0.0"); + public void testGetMessage() { + String message = "a"; + MessageVersion version = new MessageVersion("1.0.0"); AtlasNotificationMessage atlasNotificationMessage = new AtlasNotificationMessage<>(version, message); + assertEquals(atlasNotificationMessage.getMessage(), message); } @Test - public void testCompareVersion() throws Exception { + public void testCompareVersion() { MessageVersion version1 = new MessageVersion("1.0.0"); MessageVersion version2 = new MessageVersion("2.0.0"); MessageVersion version3 = new MessageVersion("0.5.0"); AtlasNotificationMessage atlasNotificationMessage = new AtlasNotificationMessage<>(version1, "a"); - assertTrue(atlasNotificationMessage.compareVersion(version1) == 0); + assertEquals(atlasNotificationMessage.compareVersion(version1), 0); assertTrue(atlasNotificationMessage.compareVersion(version2) < 0); assertTrue(atlasNotificationMessage.compareVersion(version3) > 0); } @Test - public void testMessageSource() throws Exception { - Referenceable entity = generateEntityWithTrait(); - HookNotificationV1.EntityUpdateRequest message = new HookNotificationV1.EntityUpdateRequest("user1", entity); - MessageSource source = new MessageSource(this.getClass().getSimpleName()); - List jsonList = new LinkedList<>(); + public void testMessageSource() { + Referenceable entity = generateEntityWithTrait(); + HookNotificationV1.EntityUpdateRequest message = new HookNotificationV1.EntityUpdateRequest("user1", entity); + MessageSource source = new MessageSource(this.getClass().getSimpleName()); + List jsonList = new LinkedList<>(); AbstractNotification.createNotificationMessages(message, jsonList, source); - for(Object json : jsonList) { - AtlasNotificationMessage atlasNotificationMessage = AtlasType.fromV1Json((String) json, AtlasNotificationMessage.class); - assertEquals("\"" + source.getSource() + "\"" ,AtlasType.toV1Json(atlasNotificationMessage.getSource().getSource())); + + for (String json : jsonList) { + AtlasNotificationMessage atlasNotificationMessage = AtlasType.fromV1Json(json, AtlasNotificationMessage.class); + + assertEquals("\"" + source.getSource() + "\"", AtlasType.toV1Json(atlasNotificationMessage.getSource().getSource())); } } private Referenceable generateEntityWithTrait() { - Referenceable ret = EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); - - return ret; + return EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); } - } diff --git a/notification/src/test/java/org/apache/atlas/notification/MessageVersionTest.java b/notification/src/test/java/org/apache/atlas/notification/MessageVersionTest.java index d8b3b340e9..b5f2205975 100644 --- a/notification/src/test/java/org/apache/atlas/notification/MessageVersionTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/MessageVersionTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,19 +23,22 @@ import java.util.Arrays; -import static org.testng.Assert.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; /** * MessageVersion tests. */ public class MessageVersionTest { - @Test - public void testConstructor() throws Exception { + public void testConstructor() { new MessageVersion("1.0.0"); try { new MessageVersion("foo"); + fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected @@ -43,6 +46,7 @@ public void testConstructor() throws Exception { try { new MessageVersion("A.0.0"); + fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected @@ -50,6 +54,7 @@ public void testConstructor() throws Exception { try { new MessageVersion("1.0.0a"); + fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected @@ -57,7 +62,7 @@ public void testConstructor() throws Exception { } @Test - public void testCompareTo() throws Exception { + public void testCompareTo() { MessageVersion version1 = new MessageVersion("1.0.0"); MessageVersion version2 = new MessageVersion("1.0.0"); MessageVersion version3 = new MessageVersion("2.0.0"); @@ -65,12 +70,12 @@ public void testCompareTo() throws Exception { MessageVersion version5 = new MessageVersion("1.5"); MessageVersion version6 = new MessageVersion("1.0.5"); - assertTrue(version1.compareTo(version2) == 0); - assertTrue(version2.compareTo(version1) == 0); + assertEquals(version1.compareTo(version2), 0); + assertEquals(version2.compareTo(version1), 0); assertTrue(version1.compareTo(version3) < 0); assertTrue(version3.compareTo(version1) > 0); - assertTrue(version1.compareTo(version4) == 0); - assertTrue(version4.compareTo(version1) == 0); + assertEquals(version1.compareTo(version4), 0); + assertEquals(version4.compareTo(version1), 0); assertTrue(version1.compareTo(version5) < 0); assertTrue(version5.compareTo(version1) > 0); assertTrue(version1.compareTo(version6) < 0); @@ -78,7 +83,7 @@ public void testCompareTo() throws Exception { } @Test - public void testEquals() throws Exception { + public void testEquals() { MessageVersion version1 = new MessageVersion("1.0.0"); MessageVersion version2 = new MessageVersion("1.0.0"); MessageVersion version3 = new MessageVersion("2.0.0"); @@ -86,20 +91,20 @@ public void testEquals() throws Exception { MessageVersion version5 = new MessageVersion("1.5"); MessageVersion version6 = new MessageVersion("1.0.5"); - assertTrue(version1.equals(version2)); - assertTrue(version2.equals(version1)); - assertFalse(version1.equals(version3)); - assertFalse(version3.equals(version1)); - assertTrue(version1.equals(version4)); - assertTrue(version4.equals(version1)); - assertFalse(version1.equals(version5)); - assertFalse(version5.equals(version1)); - assertFalse(version1.equals(version6)); - assertFalse(version6.equals(version1)); + assertEquals(version2, version1); + assertEquals(version1, version2); + assertNotEquals(version3, version1); + assertNotEquals(version1, version3); + assertEquals(version4, version1); + assertEquals(version1, version4); + assertNotEquals(version5, version1); + assertNotEquals(version1, version5); + assertNotEquals(version6, version1); + assertNotEquals(version1, version6); } @Test - public void testHashCode() throws Exception { + public void testHashCode() { MessageVersion version1 = new MessageVersion("1.0.0"); MessageVersion version2 = new MessageVersion("1.0.0"); MessageVersion version3 = new MessageVersion("1"); @@ -109,18 +114,21 @@ public void testHashCode() throws Exception { } @Test - public void testGetVersionParts() throws Exception { - + public void testGetVersionParts() { MessageVersion version = new MessageVersion("1.0.0"); - assertTrue(Arrays.equals(new Integer[]{1}, version.getVersionParts())); + + assertTrue(Arrays.equals(new Integer[] {1}, version.getVersionParts())); version = new MessageVersion("1.0"); - assertTrue(Arrays.equals(new Integer[]{1}, version.getVersionParts())); + + assertTrue(Arrays.equals(new Integer[] {1}, version.getVersionParts())); version = new MessageVersion("1"); - assertTrue(Arrays.equals(new Integer[]{1}, version.getVersionParts())); + + assertTrue(Arrays.equals(new Integer[] {1}, version.getVersionParts())); version = new MessageVersion("1.0.2"); - assertTrue(Arrays.equals(new Integer[]{1, 0, 2}, version.getVersionParts())); + + assertTrue(Arrays.equals(new Integer[] {1, 0, 2}, version.getVersionParts())); } } diff --git a/notification/src/test/java/org/apache/atlas/notification/RestNotificationTest.java b/notification/src/test/java/org/apache/atlas/notification/RestNotificationTest.java index 476518df68..43d32a9f16 100644 --- a/notification/src/test/java/org/apache/atlas/notification/RestNotificationTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/RestNotificationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,6 @@ import org.mockito.Matchers; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -39,7 +38,7 @@ import javax.ws.rs.core.Response; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import static org.apache.atlas.kafka.KafkaNotification.ATLAS_HOOK_TOPIC; import static org.mockito.ArgumentMatchers.anyList; @@ -47,11 +46,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class RestNotificationTest { - private NotificationInterface notifier; - private Configuration conf; + private Configuration conf; @Mock private WebResource service; @@ -69,52 +69,34 @@ public void setup() throws Exception { conf.setProperty(NotificationProvider.CONF_ATLAS_HOOK_SPOOL_ENABLED, false); notifier = NotificationProvider.get(); - - } - - private WebResource.Builder setupBuilder(AtlasClientV2.API api, WebResource webResource) { - when(webResource.path(api.getPath())).thenReturn(service); - when(webResource.path(api.getNormalizedPath())).thenReturn(service); - - return getBuilder(service); - } - - private WebResource.Builder getBuilder(WebResource resourceObject) { - when(resourceObject.getRequestBuilder()).thenReturn(resourceBuilderMock); - when(resourceObject.path(anyString())).thenReturn(resourceObject); - when(resourceBuilderMock.accept(MediaType.APPLICATION_JSON)).thenReturn(resourceBuilderMock); - when(resourceBuilderMock.type(MediaType.MULTIPART_FORM_DATA)).thenReturn(resourceBuilderMock); - when(resourceBuilderMock.type(MediaType.APPLICATION_JSON + "; charset=UTF-8")).thenReturn(resourceBuilderMock); - - return resourceBuilderMock; } @Test - public void testNotificationProvider () throws Exception { + public void testNotificationProvider() { assertEquals(notifier.getClass(), RestNotification.class); } @Test - public void testPostNotificationToTopic () throws Exception { + public void testPostNotificationToTopic() { AtlasClientV2 client = new AtlasClientV2(service, conf); AtlasBaseClient.API api = client.formatPathWithParameter(AtlasClientV2.API_V2.POST_NOTIFICATIONS_TO_TOPIC, ATLAS_HOOK_TOPIC); WebResource.Builder builder = setupBuilder(api, service); ClientResponse response = mock(ClientResponse.class); when(response.getStatus()).thenReturn(Response.Status.NO_CONTENT.getStatusCode()); - when(builder.method(anyString(), Matchers.any(), anyList())).thenReturn(response); + when(builder.method(anyString(), Matchers.>any(), anyList())).thenReturn(response); - ((RestNotification)notifier).atlasClientV2 = client; + ((RestNotification) notifier).atlasClientV2 = client; try { - ((RestNotification)notifier).sendInternal(NotificationInterface.NotificationType.HOOK, new ArrayList(Arrays.asList("Dummy"))); + ((RestNotification) notifier).sendInternal(NotificationInterface.NotificationType.HOOK, new ArrayList<>(Collections.singletonList("Dummy"))); } catch (NotificationException e) { - Assert.fail("Failed with Exception"); + fail("Failed with Exception"); } } @Test - public void testNotificationException () throws Exception { + public void testNotificationException() { AtlasClientV2 client = new AtlasClientV2(service, conf); AtlasBaseClient.API api = client.formatPathWithParameter(AtlasClientV2.API_V2.POST_NOTIFICATIONS_TO_TOPIC, ATLAS_HOOK_TOPIC); WebResource.Builder builder = setupBuilder(api, service); @@ -122,15 +104,31 @@ public void testNotificationException () throws Exception { when(response.getStatus()).thenReturn(AtlasErrorCode.NOTIFICATION_EXCEPTION.getHttpCode().getStatusCode()); when(response.getEntity(String.class)).thenReturn(AtlasErrorCode.NOTIFICATION_EXCEPTION.getErrorCode()); - when(builder.method(anyString(), Matchers.any(), anyList())).thenReturn(response); + when(builder.method(anyString(), Matchers.>any(), anyList())).thenReturn(response); - ((RestNotification)notifier).atlasClientV2 = client; + ((RestNotification) notifier).atlasClientV2 = client; try { - ((RestNotification)notifier).sendInternal(NotificationInterface.NotificationType.HOOK, new ArrayList(Arrays.asList("Dummy"))); + ((RestNotification) notifier).sendInternal(NotificationInterface.NotificationType.HOOK, new ArrayList<>(Collections.singletonList("Dummy"))); } catch (NotificationException e) { - Assert.assertTrue(e.getMessage().contains(AtlasErrorCode.NOTIFICATION_EXCEPTION.getErrorCode())); + assertTrue(e.getMessage().contains(AtlasErrorCode.NOTIFICATION_EXCEPTION.getErrorCode())); } } + private WebResource.Builder setupBuilder(AtlasClientV2.API api, WebResource webResource) { + when(webResource.path(api.getPath())).thenReturn(service); + when(webResource.path(api.getNormalizedPath())).thenReturn(service); + + return getBuilder(service); + } + + private WebResource.Builder getBuilder(WebResource resourceObject) { + when(resourceObject.getRequestBuilder()).thenReturn(resourceBuilderMock); + when(resourceObject.path(anyString())).thenReturn(resourceObject); + when(resourceBuilderMock.accept(MediaType.APPLICATION_JSON)).thenReturn(resourceBuilderMock); + when(resourceBuilderMock.type(MediaType.MULTIPART_FORM_DATA)).thenReturn(resourceBuilderMock); + when(resourceBuilderMock.type(MediaType.APPLICATION_JSON + "; charset=UTF-8")).thenReturn(resourceBuilderMock); + + return resourceBuilderMock; + } } diff --git a/notification/src/test/java/org/apache/atlas/notification/SplitMessageAggregatorTest.java b/notification/src/test/java/org/apache/atlas/notification/SplitMessageAggregatorTest.java index b79735ad89..b1448a079b 100644 --- a/notification/src/test/java/org/apache/atlas/notification/SplitMessageAggregatorTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/SplitMessageAggregatorTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,36 +19,36 @@ import org.apache.atlas.model.notification.AtlasNotificationBaseMessage.CompressionKind; import org.apache.atlas.model.notification.AtlasNotificationStringMessage; -import org.testng.Assert; import org.testng.annotations.Test; import java.util.HashMap; import java.util.Map; +import static org.testng.Assert.assertEquals; + public class SplitMessageAggregatorTest { @Test public void verifyEviction() throws InterruptedException { Map map = getStringSplitMessageAggregatorMap(); - Thread.currentThread().sleep(500); + Thread.sleep(500); AtlasNotificationMessageDeserializer.purgeStaleMessages(map, System.currentTimeMillis(), 250); - Assert.assertEquals(map.size(), 0); + assertEquals(map.size(), 0); } - @Test public void verifyEvictionDoesNotOccur() throws InterruptedException { Map map = getStringSplitMessageAggregatorMap(); int expectedSize = map.size(); - Thread.currentThread().sleep(500); + Thread.sleep(500); AtlasNotificationMessageDeserializer.purgeStaleMessages(map, System.currentTimeMillis(), Long.MAX_VALUE); - Assert.assertEquals(map.size(), expectedSize); + assertEquals(map.size(), expectedSize); } private Map getStringSplitMessageAggregatorMap() { @@ -66,7 +66,7 @@ private SplitMessageAggregator getSplitMessageAggregator(String id, int splitCou for (int i = 0; i < splitCount; i++) { AtlasNotificationStringMessage sm = new AtlasNotificationStringMessage("aaaaa", id, CompressionKind.NONE, i, splitCount); - if(sma == null) { + if (sma == null) { sma = new SplitMessageAggregator(sm); } else { sma.add(sm); diff --git a/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationDeserializerTest.java b/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationDeserializerTest.java index 2953b63db3..558419475d 100644 --- a/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationDeserializerTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationDeserializerTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,9 +20,9 @@ import org.apache.atlas.model.notification.EntityNotification; import org.apache.atlas.model.notification.MessageSource; +import org.apache.atlas.notification.AbstractNotification; import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.v1.model.instance.Struct; -import org.apache.atlas.notification.AbstractNotification; import org.apache.atlas.v1.model.notification.EntityNotificationV1; import org.testng.annotations.Test; @@ -37,14 +37,14 @@ * EntityMessageDeserializer tests. */ public class EntityNotificationDeserializerTest { - private EntityMessageDeserializer deserializer = new EntityMessageDeserializer(); MessageSource source = new MessageSource(this.getClass().getSimpleName()); + private final EntityMessageDeserializer deserializer = new EntityMessageDeserializer(); @Test public void testDeserialize() throws Exception { Referenceable entity = EntityNotificationTest.getEntity("id"); String traitName = "MyTrait"; - List traits = Collections.singletonList(new Struct(traitName, Collections.emptyMap())); + List traits = Collections.singletonList(new Struct(traitName, Collections.emptyMap())); EntityNotificationV1 notification = new EntityNotificationV1(entity, EntityNotificationV1.OperationType.TRAIT_ADD, traits); List jsonMsgList = new ArrayList<>(); @@ -53,7 +53,7 @@ public void testDeserialize() throws Exception { EntityNotification deserializedNotification = null; for (String jsonMsg : jsonMsgList) { - deserializedNotification = deserializer.deserialize(jsonMsg); + deserializedNotification = deserializer.deserialize(jsonMsg); if (deserializedNotification != null) { break; @@ -62,7 +62,7 @@ public void testDeserialize() throws Exception { assertTrue(deserializedNotification instanceof EntityNotificationV1); - EntityNotificationV1 entityNotificationV1 = (EntityNotificationV1)deserializedNotification; + EntityNotificationV1 entityNotificationV1 = (EntityNotificationV1) deserializedNotification; assertEquals(entityNotificationV1.getOperationType(), notification.getOperationType()); assertEquals(entityNotificationV1.getEntity().getId(), notification.getEntity().getId()); diff --git a/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationTest.java b/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationTest.java index 232b21da97..0dbddb8853 100644 --- a/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/entity/EntityNotificationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,10 +18,10 @@ package org.apache.atlas.notification.entity; -import org.apache.atlas.v1.model.instance.Referenceable; -import org.apache.atlas.v1.model.instance.Struct; import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.v1.model.instance.Referenceable; +import org.apache.atlas.v1.model.instance.Struct; import org.apache.atlas.v1.model.notification.EntityNotificationV1; import org.apache.atlas.v1.model.notification.EntityNotificationV1.OperationType; import org.testng.annotations.Test; @@ -42,28 +42,42 @@ * EntityNotificationV1 tests. */ public class EntityNotificationTest { + public static Referenceable getEntity(String id, Struct... traits) { + String typeName = "typeName"; + List traitNames = new LinkedList<>(); + Map traitMap = new HashMap<>(); + + for (Struct trait : traits) { + String traitName = trait.getTypeName(); + + traitNames.add(traitName); + traitMap.put(traitName, trait); + } + + return new Referenceable(id, typeName, new HashMap<>(), traitNames, traitMap); + } @Test - public void testGetEntity() throws Exception { + public void testGetEntity() { Referenceable entity = getEntity("id"); - EntityNotificationV1 entityNotification = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); + EntityNotificationV1 entityNotification = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); assertEquals(entity, entityNotification.getEntity()); } @Test - public void testGetOperationType() throws Exception { + public void testGetOperationType() { Referenceable entity = getEntity("id"); - EntityNotificationV1 entityNotification = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); + EntityNotificationV1 entityNotification = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); - assertEquals(EntityNotificationV1.OperationType.ENTITY_CREATE, entityNotification.getOperationType()); + assertEquals(entityNotification.getOperationType(), OperationType.ENTITY_CREATE); } @Test - public void testGetAllTraits() throws Exception { + public void testGetAllTraits() { Referenceable entity = getEntity("id"); String traitName = "MyTrait"; - List traitInfo = Collections.singletonList(new Struct(traitName, Collections.emptyMap())); + List traitInfo = Collections.singletonList(new Struct(traitName, Collections.emptyMap())); EntityNotificationV1 entityNotification = new EntityNotificationV1(entity, OperationType.TRAIT_ADD, traitInfo); @@ -71,7 +85,7 @@ public void testGetAllTraits() throws Exception { } @Test - public void testGetAllTraitsSuperTraits() throws Exception { + public void testGetAllTraitsSuperTraits() { AtlasTypeRegistry typeRegistry = mock(AtlasTypeRegistry.class); String traitName = "MyTrait"; Struct myTrait = new Struct(traitName); @@ -92,7 +106,7 @@ public void testGetAllTraitsSuperTraits() throws Exception { List allTraits = entityNotification.getAllTraits(); - assertEquals(2, allTraits.size()); + assertEquals(allTraits.size(), 2); for (Struct trait : allTraits) { String typeName = trait.getTypeName(); @@ -102,27 +116,12 @@ public void testGetAllTraitsSuperTraits() throws Exception { } @Test - public void testEquals() throws Exception { + public void testEquals() { Referenceable entity = getEntity("id"); - EntityNotificationV1 entityNotification2 = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); - EntityNotificationV1 entityNotification = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); - - assertTrue(entityNotification.equals(entityNotification2)); - assertTrue(entityNotification2.equals(entityNotification)); - } - - public static Referenceable getEntity(String id, Struct... traits) { - String typeName = "typeName"; - List traitNames = new LinkedList<>(); - Map traitMap = new HashMap<>(); - - for (Struct trait : traits) { - String traitName = trait.getTypeName(); - - traitNames.add(traitName); - traitMap.put(traitName, trait); - } + EntityNotificationV1 entityNotification2 = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); + EntityNotificationV1 entityNotification = new EntityNotificationV1(entity, OperationType.ENTITY_CREATE, Collections.emptyList()); - return new Referenceable(id, typeName, new HashMap(), traitNames, traitMap); + assertEquals(entityNotification2, entityNotification); + assertEquals(entityNotification, entityNotification2); } } diff --git a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationDeserializerTest.java b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationDeserializerTest.java index bfc9b531db..cce2749260 100644 --- a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationDeserializerTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationDeserializerTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,14 +18,14 @@ package org.apache.atlas.notification.hook; -import org.apache.atlas.model.notification.MessageSource; import org.apache.atlas.model.notification.HookNotification; +import org.apache.atlas.model.notification.MessageSource; +import org.apache.atlas.notification.AbstractNotification; import org.apache.atlas.notification.entity.EntityNotificationTest; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.v1.model.instance.Struct; -import org.apache.atlas.notification.AbstractNotification; import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityUpdateRequest; -import org.apache.atlas.type.AtlasType; import org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.Test; @@ -41,8 +41,8 @@ * HookMessageDeserializer tests. */ public class HookNotificationDeserializerTest { - private HookMessageDeserializer deserializer = new HookMessageDeserializer(); MessageSource source = new MessageSource(this.getClass().getSimpleName()); + private final HookMessageDeserializer deserializer = new HookMessageDeserializer(); @Test public void testDeserialize() throws Exception { @@ -70,13 +70,13 @@ public void testDeserializeLegacyMessage() throws Exception { @Test public void testDeserializeCompressedMessage() throws Exception { - Referenceable entity = generateLargeEntityWithTrait(); - EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); - List jsonMsgList = new ArrayList<>(); + Referenceable entity = generateLargeEntityWithTrait(); + EntityUpdateRequest message = new EntityUpdateRequest("user1", entity); + List jsonMsgList = new ArrayList<>(); AbstractNotification.createNotificationMessages(message, jsonMsgList, source); - assertTrue(jsonMsgList.size() == 1); + assertEquals(jsonMsgList.size(), 1); String compressedMsg = jsonMsgList.get(0); String uncompressedMsg = AtlasType.toV1Json(message); @@ -104,7 +104,7 @@ public void testDeserializeSplitMessage() throws Exception { } private Referenceable generateEntityWithTrait() { - Referenceable ret = EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); + Referenceable ret = EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); return ret; } @@ -139,11 +139,10 @@ private void assertEqualMessage(HookNotification deserializedMessage, EntityUpda assertEquals(deserializedEntity.getTypeName(), entity.getTypeName()); assertEquals(deserializedEntity.getTraits(), entity.getTraits()); assertEquals(deserializedEntity.getTrait(traitName).hashCode(), entity.getTrait(traitName).hashCode()); - } private Referenceable generateLargeEntityWithTrait() { - Referenceable ret = EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); + Referenceable ret = EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); // add 100 attributes, each with value of size 10k // Json Size=1,027,984; GZipped Size=16,387 ==> will compress, but not split @@ -156,7 +155,7 @@ private Referenceable generateLargeEntityWithTrait() { } private Referenceable generateVeryLargeEntityWithTrait() { - Referenceable ret = EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); + Referenceable ret = EntityNotificationTest.getEntity("id", new Struct("MyTrait", Collections.emptyMap())); // add 300 attributes, each with value of size 10k // Json Size=3,082,384; GZipped Size=2,313,357 ==> will compress & split diff --git a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java index ccfd264745..f341e96509 100644 --- a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,15 +24,14 @@ import org.apache.atlas.model.notification.HookNotification; import org.apache.atlas.model.notification.HookNotification.EntityCreateRequestV2; import org.apache.atlas.model.notification.HookNotification.EntityDeleteRequestV2; -import org.apache.atlas.model.notification.HookNotification.EntityUpdateRequestV2; import org.apache.atlas.model.notification.HookNotification.EntityPartialUpdateRequestV2; +import org.apache.atlas.model.notification.HookNotification.EntityUpdateRequestV2; import org.apache.atlas.model.notification.HookNotification.HookNotificationType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityCreateRequest; - import org.testng.annotations.Test; import java.util.ArrayList; @@ -43,14 +42,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; - public class HookNotificationTest { - private HookMessageDeserializer deserializer = new HookMessageDeserializer(); - private static final String ATTR_VALUE_STRING = "strValue"; private static final Integer ATTR_VALUE_INTEGER = 10; private static final Boolean ATTR_VALUE_BOOLEAN = Boolean.TRUE; - private static final Date ATTR_VALUE_DATE = new Date(); + private static final Date ATTR_VALUE_DATE = new Date(); + private final HookMessageDeserializer deserializer = new HookMessageDeserializer(); @Test public void testNewMessageSerDe() throws Exception { @@ -62,8 +59,8 @@ public void testNewMessageSerDe() throws Exception { String user = "user"; - EntityCreateRequest request = new EntityCreateRequest(user, entity1, entity2); - String notificationJson = AtlasType.toV1Json(request); + EntityCreateRequest request = new EntityCreateRequest(user, entity1, entity2); + String notificationJson = AtlasType.toV1Json(request); HookNotification actualNotification = deserializer.deserialize(notificationJson); assertEquals(actualNotification.getType(), HookNotificationType.ENTITY_CREATE); @@ -77,7 +74,7 @@ public void testNewMessageSerDe() throws Exception { Referenceable actualEntity1 = createRequest.getEntities().get(0); assertEquals(actualEntity1.getTypeName(), "sometype"); - assertEquals(((Referenceable)actualEntity1.get("complex")).getTypeName(), "othertype"); + assertEquals(((Referenceable) actualEntity1.get("complex")).getTypeName(), "othertype"); assertEquals(createRequest.getEntities().get(1).getTypeName(), "newtype"); } @@ -115,7 +112,6 @@ public void testBackwardCompatibility() throws Exception { + " \"type\": \"ENTITY_CREATE\"\n" + "}"; - HookNotification actualNotification = deserializer.deserialize(notificationJson); assertEquals(actualNotification.getType(), HookNotificationType.ENTITY_CREATE); @@ -148,10 +144,10 @@ public void testEntityCreateV2SerDe() throws Exception { assertEquals(createRequest.getEntities().getEntities().size(), 2); - AtlasEntity actualEntity1 = createRequest.getEntities().getEntities().get(0); - AtlasEntity actualEntity2 = createRequest.getEntities().getEntities().get(1); - AtlasEntity actualEntity3 = createRequest.getEntities().getReferredEntity(entity3.getGuid()); - Map actualComplexAttr = (Map)actualEntity1.getAttribute("complex"); + AtlasEntity actualEntity1 = createRequest.getEntities().getEntities().get(0); + AtlasEntity actualEntity2 = createRequest.getEntities().getEntities().get(1); + AtlasEntity actualEntity3 = createRequest.getEntities().getReferredEntity(entity3.getGuid()); + Map actualComplexAttr = (Map) actualEntity1.getAttribute("complex"); assertEquals(actualEntity1.getGuid(), entity1.getGuid()); assertEquals(actualEntity1.getTypeName(), entity1.getTypeName()); @@ -192,10 +188,10 @@ public void testEntityUpdateV2SerDe() throws Exception { assertEquals(updateRequest.getEntities().getEntities().size(), 2); - AtlasEntity actualEntity1 = updateRequest.getEntities().getEntities().get(0); - AtlasEntity actualEntity2 = updateRequest.getEntities().getEntities().get(1); - AtlasEntity actualEntity3 = updateRequest.getEntities().getReferredEntity(entity3.getGuid()); - Map actualComplexAttr = (Map)actualEntity1.getAttribute("complex"); + AtlasEntity actualEntity1 = updateRequest.getEntities().getEntities().get(0); + AtlasEntity actualEntity2 = updateRequest.getEntities().getEntities().get(1); + AtlasEntity actualEntity3 = updateRequest.getEntities().getReferredEntity(entity3.getGuid()); + Map actualComplexAttr = (Map) actualEntity1.getAttribute("complex"); assertEquals(actualEntity1.getGuid(), entity1.getGuid()); assertEquals(actualEntity1.getTypeName(), entity1.getTypeName()); @@ -235,10 +231,10 @@ public void testEntityPartialUpdateV2SerDe() throws Exception { assertEquals(updateRequest.getEntity().getReferredEntities().size(), 2); - AtlasEntity actualEntity1 = updateRequest.getEntity().getEntity(); - AtlasEntity actualEntity2 = updateRequest.getEntity().getReferredEntity(entity2.getGuid()); - AtlasEntity actualEntity3 = updateRequest.getEntity().getReferredEntity(entity3.getGuid()); - Map actualComplexAttr = (Map)actualEntity1.getAttribute("complex"); + AtlasEntity actualEntity1 = updateRequest.getEntity().getEntity(); + AtlasEntity actualEntity2 = updateRequest.getEntity().getReferredEntity(entity2.getGuid()); + AtlasEntity actualEntity3 = updateRequest.getEntity().getReferredEntity(entity3.getGuid()); + Map actualComplexAttr = (Map) actualEntity1.getAttribute("complex"); assertEquals(actualEntity1.getGuid(), entity1.getGuid()); assertEquals(actualEntity1.getTypeName(), entity1.getTypeName()); diff --git a/notification/src/test/java/org/apache/atlas/notification/spool/AtlasFileSpoolTest.java b/notification/src/test/java/org/apache/atlas/notification/spool/AtlasFileSpoolTest.java index 265598eeb7..37fd165a28 100644 --- a/notification/src/test/java/org/apache/atlas/notification/spool/AtlasFileSpoolTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/spool/AtlasFileSpoolTest.java @@ -23,7 +23,6 @@ import org.apache.atlas.notification.NotificationException; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomUtils; -import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -39,66 +38,19 @@ import static org.testng.Assert.assertTrue; public class AtlasFileSpoolTest extends BaseTest { - private static int MAX_RECORDS = 50; - - private static class MessageHandlerSpy extends AbstractNotification { - - private List publishedMessages = new ArrayList<>(); - - public List getMessages() { - return publishedMessages; - } - - @Override - public void init(String source, Object failedMessagesLogger) { - } - - @Override - public void setCurrentUser(String user) { - - } - - @Override - public void sendInternal(NotificationType type, List messages) throws NotificationException { - publishedMessages.addAll(messages); - - } - - @Override - public List> createConsumers(NotificationType notificationType, int numConsumers) { - return null; - } - - @Override - public void send(NotificationType type, T... messages) throws NotificationException { - } - - @Override - public void send(NotificationType type, List messages) throws NotificationException { - } - - @Override - public void close() { - - } - - @Override - public boolean isReady(NotificationType type) { - return true; - } - } + private static final int MAX_RECORDS = 50; @Test public void indexSetupMultipleTimes() throws IOException, AtlasException { - SpoolConfiguration cfg = getSpoolConfiguration(); - IndexManagement indexManagement = new IndexManagement(cfg); + SpoolConfiguration cfg = getSpoolConfiguration(); + IndexManagement indexManagement = new IndexManagement(cfg); for (int i = 0; i < 2; i++) { indexManagement.init(); assertTrue(cfg.getSpoolDir().exists()); assertTrue(cfg.getArchiveDir().exists()); - File indexFile = indexManagement.getIndexFileManager().getIndexFile(); + File indexFile = indexManagement.getIndexFileManager().getIndexFile(); File indexDoneFile = indexManagement.getIndexFileManager().getDoneFile(); assertTrue(indexFile.exists(), "File not created: " + indexFile.getAbsolutePath()); @@ -108,8 +60,8 @@ public void indexSetupMultipleTimes() throws IOException, AtlasException { @Test public void spoolerTest() throws IOException, AtlasException { - SpoolConfiguration cfg = getSpoolConfigurationTest(); - IndexManagement indexManagement = new IndexManagement(cfg); + SpoolConfiguration cfg = getSpoolConfigurationTest(); + IndexManagement indexManagement = new IndexManagement(cfg); indexManagement.init(); Spooler spooler = new Spooler(cfg, indexManagement); @@ -128,73 +80,120 @@ public void publisherTest() throws IOException, AtlasException, InterruptedExcep indexManagement.init(); MessageHandlerSpy messageHandler = new MessageHandlerSpy(); - Publisher publisher = new Publisher(cfg, indexManagement, messageHandler); - boolean ret = publisher.processAndDispatch(indexManagement.getIndexFileManager().getRecords().get(0)); + Publisher publisher = new Publisher(cfg, indexManagement, messageHandler); + boolean ret = publisher.processAndDispatch(indexManagement.getIndexFileManager().getRecords().get(0)); publisher.setDrain(); - Assert.assertTrue(ret); + assertTrue(ret); TimeUnit.SECONDS.sleep(5); + assertTrue(messageHandler.getMessages().size() >= 0); } @Test public void indexRecordsRead() throws IOException, AtlasException { - SpoolConfiguration spoolCfg = getSpoolConfigurationTest(); - IndexManagement indexManagement = new IndexManagement(spoolCfg); + SpoolConfiguration spoolCfg = getSpoolConfigurationTest(); + IndexManagement indexManagement = new IndexManagement(spoolCfg); indexManagement.init(); - } @Test public void concurrentWriteAndPublish() throws InterruptedException, IOException, AtlasException { - final int MAX_PROCESSES = 4; - SpoolConfiguration spoolCfg = getSpoolConfigurationTest(5); + final int maxProcesses = 4; + SpoolConfiguration spoolCfg = getSpoolConfigurationTest(5); - IndexManagement[] im1 = new IndexManagement[MAX_PROCESSES]; - MessageHandlerSpy[] messageHandlerSpy = new MessageHandlerSpy[MAX_PROCESSES]; + IndexManagement[] im1 = new IndexManagement[maxProcesses]; + MessageHandlerSpy[] messageHandlerSpy = new MessageHandlerSpy[maxProcesses]; - for (int i = 0; i < MAX_PROCESSES; i++) { + for (int i = 0; i < maxProcesses; i++) { messageHandlerSpy[i] = new MessageHandlerSpy(); - im1[i] = new IndexManagement(spoolCfg); + im1[i] = new IndexManagement(spoolCfg); } - for (int i = 0; i < MAX_PROCESSES; i++) { + for (int i = 0; i < maxProcesses; i++) { im1[i].init(); } IndexManagement imVerify = new IndexManagement(spoolCfg); imVerify.init(); - Assert.assertTrue(imVerify.getIndexFileManager().getRecords().size() >= 0); + assertTrue(imVerify.getIndexFileManager().getRecords().size() >= 0); - Thread[] th1 = new Thread[MAX_PROCESSES]; - for (int i = 0; i < MAX_PROCESSES; i++) { + Thread[] th1 = new Thread[maxProcesses]; + for (int i = 0; i < maxProcesses; i++) { th1[i] = new Thread(new MessagePump(new Spooler(spoolCfg, im1[i]), new Publisher(spoolCfg, im1[i], messageHandlerSpy[i]))); } - for (int i = 0; i < MAX_PROCESSES; i++) { + for (int i = 0; i < maxProcesses; i++) { th1[i].start(); } - for (int i = 0; i < MAX_PROCESSES; i++) { + for (int i = 0; i < maxProcesses; i++) { th1[i].join(); } imVerify = new IndexManagement(spoolCfg); imVerify.init(); - Assert.assertEquals(imVerify.getIndexFileManager().getRecords().size(), 0); - for (int i = 0; i < MAX_PROCESSES; i++) { - Assert.assertTrue(messageHandlerSpy[i].getMessages().size() >= 0); + assertEquals(imVerify.getIndexFileManager().getRecords().size(), 0); + for (int i = 0; i < maxProcesses; i++) { + assertTrue(messageHandlerSpy[i].getMessages().size() >= 0); } } - private class MessagePump implements Runnable { + @AfterClass + public void tearDown() { + FileUtils.deleteQuietly(new File(spoolDirTest)); + } + + private static class MessageHandlerSpy extends AbstractNotification { + private final List publishedMessages = new ArrayList<>(); + + public List getMessages() { + return publishedMessages; + } + + @Override + public void init(String source, Object failedMessagesLogger) { + } + + @Override + public void sendInternal(NotificationType type, List messages) { + publishedMessages.addAll(messages); + } + + @Override + public void setCurrentUser(String user) { + } + + @Override + public void send(NotificationType type, T... messages) { + } - private Spooler spooler; - private Publisher publisher; - private Thread publisherThread; + @Override + public void send(NotificationType type, List messages) { + } + + @Override + public List> createConsumers(NotificationType notificationType, int numConsumers) { + return null; + } + + @Override + public void close() { + } + + @Override + public boolean isReady(NotificationType type) { + return true; + } + } + + private static class MessagePump implements Runnable { + private final Spooler spooler; + private final Publisher publisher; + private Thread publisherThread; public MessagePump(Spooler spooler, Publisher publisher) { - this.spooler = spooler; + this.spooler = spooler; this.publisher = publisher; } @@ -208,10 +207,8 @@ public void run() { spooler.send(HOOK, String.format("%s-%s", "message", i)); Thread.sleep(RandomUtils.nextInt(10, 100)); - } catch (NotificationException exception) { + } catch (NotificationException | InterruptedException exception) { exception.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); } } @@ -224,10 +221,4 @@ public void run() { } } } - - - @AfterClass - public void tearDown() { - FileUtils.deleteQuietly(new File(spoolDirTest)); - } } diff --git a/notification/src/test/java/org/apache/atlas/notification/spool/BaseTest.java b/notification/src/test/java/org/apache/atlas/notification/spool/BaseTest.java index 83971f6af8..2dff8ccf18 100644 --- a/notification/src/test/java/org/apache/atlas/notification/spool/BaseTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/spool/BaseTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,16 +25,17 @@ import java.io.IOException; public class BaseTest { - public static String spoolDir = System.getProperty("user.dir") + "/src/test/resources/spool"; - public static String spoolDirTest = spoolDir + "-test"; - protected final String SOURCE_TEST = "test-src"; - protected final String SOURCE_TEST_HANDLER = "1"; + public static String spoolDir = System.getProperty("user.dir") + "/src/test/resources/spool"; + public static String spoolDirTest = spoolDir + "-test"; - protected final String knownIndexFilePath = "index-test-src-1.json"; + protected static final String SOURCE_TEST = "test-src"; + protected static final String SOURCE_TEST_HANDLER = "1"; + + protected final String knownIndexFilePath = "index-test-src-1.json"; protected final String knownIndexDoneFilePath = "index-test-src-1_closed.json"; - protected File archiveDir = new File(spoolDir, "archive"); - protected File indexFile = new File(spoolDir, knownIndexFilePath); + protected File archiveDir = new File(spoolDir, "archive"); + protected File indexFile = new File(spoolDir, knownIndexFilePath); protected File indexDoneFile = new File(spoolDir, knownIndexDoneFilePath); public SpoolConfiguration getSpoolConfiguration() { @@ -44,6 +45,7 @@ public SpoolConfiguration getSpoolConfiguration() { public SpoolConfiguration getSpoolConfigurationTest() { return getSpoolConfiguration(spoolDirTest, SOURCE_TEST_HANDLER); } + public SpoolConfiguration getSpoolConfigurationTest(Integer testId) { return getSpoolConfiguration(spoolDirTest, testId.toString()); } diff --git a/notification/src/test/java/org/apache/atlas/notification/spool/IndexManagementTest.java b/notification/src/test/java/org/apache/atlas/notification/spool/IndexManagementTest.java index f9d2a0670e..8a55302ca3 100644 --- a/notification/src/test/java/org/apache/atlas/notification/spool/IndexManagementTest.java +++ b/notification/src/test/java/org/apache/atlas/notification/spool/IndexManagementTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,7 +21,6 @@ import org.apache.atlas.notification.spool.models.IndexRecords; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; -import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -31,39 +30,45 @@ import java.util.List; import java.util.Set; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + public class IndexManagementTest extends BaseTest { @Test public void fileNameGeneration() { - String handlerName = "someHandler"; - SpoolConfiguration cfg = getSpoolConfiguration(spoolDir, handlerName); + String handlerName = "someHandler"; + SpoolConfiguration cfg = getSpoolConfiguration(spoolDir, handlerName); + IndexRecord record = new IndexRecord(StringUtils.EMPTY); - IndexRecord record = new IndexRecord(StringUtils.EMPTY); - Assert.assertEquals(SpoolUtils.getIndexFileName(cfg.getSourceName(), cfg.getMessageHandlerName()), "index-test-src-someHandler.json"); - Assert.assertTrue(SpoolUtils.getSpoolFileName(cfg.getSourceName(), cfg.getMessageHandlerName(), record.getId()).startsWith("spool-test-src-someHandler-")); + assertEquals(SpoolUtils.getIndexFileName(cfg.getSourceName(), cfg.getMessageHandlerName()), "index-test-src-someHandler.json"); + assertTrue(SpoolUtils.getSpoolFileName(cfg.getSourceName(), cfg.getMessageHandlerName(), record.getId()).startsWith("spool-test-src-someHandler-")); } @Test public void verifyLoad() throws IOException { - final int expectedRecords = 2; - SpoolConfiguration cfg = getSpoolConfiguration(); + final int expectedRecords = 2; + SpoolConfiguration cfg = getSpoolConfiguration(); IndexManagement.IndexFileManager indexFileManager = new IndexManagement.IndexFileManager(SOURCE_TEST, cfg.getIndexFile(), cfg.getIndexDoneFile(), null, 2); - Assert.assertEquals(indexFileManager.getRecords().size(), expectedRecords); + assertEquals(indexFileManager.getRecords().size(), expectedRecords); - Assert.assertEquals(indexFileManager.getRecords().get(0).getId(), "1"); - Assert.assertEquals(indexFileManager.getRecords().get(1).getId(), "2"); + assertEquals(indexFileManager.getRecords().get(0).getId(), "1"); + assertEquals(indexFileManager.getRecords().get(1).getId(), "2"); } @Test public void addAndRemove() throws IOException { - File newIndexFile = getNewIndexFile('3'); + File newIndexFile = getNewIndexFile('3'); File newIndexDoneFile = getNewIndexDoneFile('3'); IndexManagement.IndexFileManager indexFileManager = new IndexManagement.IndexFileManager(SOURCE_TEST, newIndexFile, newIndexDoneFile, null, 2); int expectedCount = 2; - Assert.assertEquals(indexFileManager.getRecords().size(), expectedCount); + + assertEquals(indexFileManager.getRecords().size(), expectedCount); IndexRecord r3 = indexFileManager.add("3.log"); IndexRecord r4 = indexFileManager.add("4.log"); @@ -76,29 +81,29 @@ public void addAndRemove() throws IOException { indexFileManager.updateIndex(r5); IndexRecords records = indexFileManager.loadRecords(newIndexFile); - Assert.assertTrue(records.getRecords().containsKey(r3.getId())); - Assert.assertTrue(records.getRecords().containsKey(r4.getId())); - Assert.assertTrue(records.getRecords().containsKey(r5.getId())); + assertTrue(records.getRecords().containsKey(r3.getId())); + assertTrue(records.getRecords().containsKey(r4.getId())); + assertTrue(records.getRecords().containsKey(r5.getId())); - Assert.assertEquals(records.getRecords().get(r3.getId()).getStatus(), r3.getStatus()); - Assert.assertEquals(records.getRecords().get(r4.getId()).getFailedAttempt(), r4.getFailedAttempt()); - Assert.assertEquals(records.getRecords().get(r5.getId()).getLine(), r5.getLine()); + assertEquals(records.getRecords().get(r3.getId()).getStatus(), r3.getStatus()); + assertEquals(records.getRecords().get(r4.getId()).getFailedAttempt(), r4.getFailedAttempt()); + assertEquals(records.getRecords().get(r5.getId()).getLine(), r5.getLine()); indexFileManager.remove(r3); indexFileManager.remove(r4); indexFileManager.remove(r5); - Assert.assertEquals(indexFileManager.getRecords().size(), expectedCount); + assertEquals(indexFileManager.getRecords().size(), expectedCount); } @Test public void verifyOperations() throws IOException { SpoolConfiguration cfg = getSpoolConfigurationTest(); - File newIndexFile = getNewIndexFile('2'); + File newIndexFile = getNewIndexFile('2'); File newIndexDoneFile = getNewIndexDoneFile('2'); - File archiveDir = cfg.getArchiveDir(); + File archiveDir = cfg.getArchiveDir(); IndexManagement.IndexFileManager indexFileManager = new IndexManagement.IndexFileManager(SOURCE_TEST, newIndexFile, newIndexDoneFile, null, 2); verifyAdding(indexFileManager); @@ -111,16 +116,21 @@ public void verifyOperations() throws IOException { verifyArchiving(indexFileManager); } + @AfterClass + public void tearDown() { + FileUtils.deleteQuietly(new File(spoolDirTest)); + } + private void verifyRecords(IndexManagement.IndexFileManager indexFileManager) { List records = indexFileManager.getRecords(); - Assert.assertEquals(records.size(), 5); - Assert.assertTrue(records.get(3).getPath().endsWith("3.log")); - Assert.assertEquals(records.get(3).getStatus(), IndexRecord.STATUS_WRITE_IN_PROGRESS); - Assert.assertEquals(records.get(2).getFailedAttempt(), 0); - Assert.assertEquals(records.get(1).getDoneCompleted(), 0); - Assert.assertEquals(records.get(0).getLine(), 0); - Assert.assertFalse(records.get(0).getLastSuccess() != 0); + assertEquals(records.size(), 5); + assertTrue(records.get(3).getPath().endsWith("3.log")); + assertEquals(records.get(3).getStatus(), IndexRecord.STATUS_WRITE_IN_PROGRESS); + assertEquals(records.get(2).getFailedAttempt(), 0); + assertEquals(records.get(1).getDoneCompleted(), 0); + assertEquals(records.get(0).getLine(), 0); + assertFalse(records.get(0).getLastSuccess() != 0); } private void verifyAdding(IndexManagement.IndexFileManager indexFileManager) throws IOException { @@ -143,20 +153,22 @@ private void verifyRemove(IndexManagement.IndexFileManager indexFileManager) thr indexFileManager.remove(indexFileManager.getRecords().get(5)); boolean isPending = indexFileManager.getRecords().size() > 0; - Assert.assertTrue(isPending); + + assertTrue(isPending); } private void verifySaveAndLoad(IndexManagement.IndexFileManager indexFileManager) throws IOException { indexFileManager.getRecords().get(2).updateFailedAttempt(); indexFileManager.getRecords().get(3).setDone(); - indexFileManager.getRecords().get(1).setDoneCompleted(333l); + indexFileManager.getRecords().get(1).setDoneCompleted(333L); indexFileManager.getRecords().get(0).setCurrentLine(999); - Assert.assertEquals(indexFileManager.getRecords().size(), 6); + assertEquals(indexFileManager.getRecords().size(), 6); } private void checkArchiveDir(File archiveDir) { Set availableFiles = new HashSet<>(); + availableFiles.add(new File(archiveDir, "3.log").toString()); availableFiles.add(new File(archiveDir, "4.log").toString()); @@ -165,25 +177,23 @@ private void checkArchiveDir(File archiveDir) { } File[] files = archiveDir.listFiles(); - Assert.assertNotNull(files); - Assert.assertEquals(files.length, 1); + + assertNotNull(files); + assertEquals(files.length, 1); } private void addFile(IndexManagement.IndexFileManager indexFileManager, String dir, String fileName) throws IOException { File file = new File(dir, fileName); + file.createNewFile(); + indexFileManager.add(file.toString()); } private void checkDoneFile(File newIndexDoneFile, File archiveDir, int maxArchiveFiles, String expectedFilePath) throws IOException { IndexManagement.IndexFileManager indexFileManager = new IndexManagement.IndexFileManager(SOURCE_TEST, newIndexDoneFile, newIndexDoneFile, null, maxArchiveFiles); - Assert.assertEquals(indexFileManager.getRecords().size(), 2); - Assert.assertTrue(indexFileManager.getRecords().get(1).getPath().endsWith(expectedFilePath)); - } - - @AfterClass - public void tearDown() { - FileUtils.deleteQuietly(new File(spoolDirTest)); + assertEquals(indexFileManager.getRecords().size(), 2); + assertTrue(indexFileManager.getRecords().get(1).getPath().endsWith(expectedFilePath)); } }