-
Notifications
You must be signed in to change notification settings - Fork 213
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Jira source plugin (#5125) Signed-off-by: Santhosh Gandhe <[email protected]> Signed-off-by: Maxwell Brown <[email protected]> Signed-off-by: Maxwell Brown <[email protected]> Co-authored-by: Maxwell Brown <[email protected]> Co-authored-by: Maxwell Brown <[email protected]>
- Loading branch information
3 people
authored
Nov 4, 2024
1 parent
5108528
commit 92428be
Showing
57 changed files
with
3,455 additions
and
73 deletions.
There are no files selected for viewing
3 changes: 2 additions & 1 deletion
3
data-prepper-plugins/saas-source-plugins/jira-source/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,9 @@ | ||
|
|
||
| # Metrics | ||
|
|
||
| ### Counter | ||
|
|
||
| - `issuesRequested`: measures total number of issue Requests sent. | ||
|
|
||
| ### Timer | ||
|
|
||
| - `requestProcessDuration`: measures latency of requests processed by the jira source plugin. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
138 changes: 138 additions & 0 deletions
138
...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| package org.opensearch.dataprepper.plugins.source.jira; | ||
|
|
||
| import lombok.Getter; | ||
| import lombok.Setter; | ||
| import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; | ||
| import org.opensearch.dataprepper.plugins.source.jira.utils.Constants; | ||
| import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType; | ||
| import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.UUID; | ||
|
|
||
| import static org.opensearch.dataprepper.plugins.source.jira.JiraService.CONTENT_TYPE; | ||
| import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; | ||
| import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY; | ||
| import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY; | ||
| import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME; | ||
| import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; | ||
| import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE; | ||
|
|
||
| @Setter | ||
| @Getter | ||
| public class JiraItemInfo implements ItemInfo { | ||
| private String project; | ||
| private String issueType; | ||
| private String id; | ||
| private String itemId; | ||
| private Map<String, Object> metadata; | ||
| private Instant eventTime; | ||
|
|
||
| public JiraItemInfo(String id, | ||
| String itemId, | ||
| String project, | ||
| String issueType, | ||
| Map<String, Object> metadata, | ||
| Instant eventTime | ||
| ) { | ||
| this.id = id; | ||
| this.project = project; | ||
| this.issueType = issueType; | ||
| this.itemId = itemId; | ||
| this.metadata = metadata; | ||
| this.eventTime = eventTime; | ||
| } | ||
|
|
||
| public static JiraItemInfoBuilder builder() { | ||
| return new JiraItemInfoBuilder(); | ||
| } | ||
|
|
||
| @Override | ||
| public String getPartitionKey() { | ||
| return project + "|" + issueType + "|" + UUID.randomUUID(); | ||
| } | ||
|
|
||
| @Override | ||
| public String getId() { | ||
| return id; | ||
| } | ||
|
|
||
| @Override | ||
| public Map<String, Object> getKeyAttributes() { | ||
| return Map.of(Constants.PROJECT, project); | ||
| } | ||
|
|
||
| @Override | ||
| public Instant getLastModifiedAt() { | ||
| long updatedAtMillis = Long.parseLong((String) this.metadata.getOrDefault(Constants.UPDATED, "0")); | ||
| long createdAtMillis = Long.parseLong((String) this.metadata.getOrDefault(Constants.CREATED, "0")); | ||
| return createdAtMillis > updatedAtMillis ? | ||
| Instant.ofEpochMilli(createdAtMillis) : Instant.ofEpochMilli(updatedAtMillis); | ||
| } | ||
|
|
||
| public static class JiraItemInfoBuilder { | ||
| private Map<String, Object> metadata; | ||
| private Instant eventTime; | ||
| private String id; | ||
| private String itemId; | ||
| private String project; | ||
| private String issueType; | ||
|
|
||
| public JiraItemInfoBuilder() { | ||
| } | ||
|
|
||
| public JiraItemInfo build() { | ||
| return new JiraItemInfo(id, itemId, project, issueType, metadata, eventTime); | ||
| } | ||
|
|
||
| public JiraItemInfoBuilder withMetadata(Map<String, Object> metadata) { | ||
| this.metadata = metadata; | ||
| return this; | ||
| } | ||
|
|
||
| public JiraItemInfoBuilder withEventTime(Instant eventTime) { | ||
| this.eventTime = eventTime; | ||
| return this; | ||
| } | ||
|
|
||
| public JiraItemInfoBuilder withItemId(String itemId) { | ||
| this.itemId = itemId; | ||
| return this; | ||
| } | ||
|
|
||
| public JiraItemInfoBuilder withId(String id) { | ||
| this.id = id; | ||
| return this; | ||
| } | ||
|
|
||
| public JiraItemInfoBuilder withProject(String project) { | ||
| this.project = project; | ||
| return this; | ||
| } | ||
|
|
||
| public JiraItemInfoBuilder withIssueType(String issueType) { | ||
| this.issueType = issueType; | ||
| return this; | ||
| } | ||
|
|
||
| public JiraItemInfoBuilder withIssueBean(IssueBean issue) { | ||
| Map<String, Object> issueMetadata = new HashMap<>(); | ||
| issueMetadata.put(PROJECT_KEY, issue.getProject()); | ||
| issueMetadata.put(PROJECT_NAME, issue.getProjectName()); | ||
| issueMetadata.put(CREATED, issue.getCreatedTimeMillis()); | ||
| issueMetadata.put(UPDATED, issue.getUpdatedTimeMillis()); | ||
| issueMetadata.put(ISSUE_KEY, issue.getKey()); | ||
| issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType()); | ||
|
|
||
| this.project = issue.getProject(); | ||
| this.id = issue.getKey(); | ||
| this.issueType = JiraContentType.ISSUE.getType(); | ||
| this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); | ||
| this.metadata = issueMetadata; | ||
| return this; | ||
| } | ||
| } | ||
|
|
||
| } |
64 changes: 64 additions & 0 deletions
64
...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package org.opensearch.dataprepper.plugins.source.jira; | ||
|
|
||
|
|
||
| import lombok.Setter; | ||
| import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; | ||
| import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import javax.inject.Named; | ||
| import java.time.Instant; | ||
| import java.util.Iterator; | ||
| import java.util.Queue; | ||
| import java.util.concurrent.ConcurrentLinkedQueue; | ||
| import java.util.concurrent.ExecutorService; | ||
|
|
||
| @Named | ||
| public class JiraIterator implements Iterator<ItemInfo> { | ||
|
|
||
| private static final Logger log = LoggerFactory.getLogger(JiraIterator.class); | ||
| private final JiraSourceConfig sourceConfig; | ||
| private final JiraService service; | ||
| private final ExecutorService crawlerTaskExecutor; | ||
| @Setter | ||
| private long crawlerQWaitTimeMillis = 2000; | ||
| private Queue<ItemInfo> itemInfoQueue; | ||
| private Instant lastPollTime; | ||
| private boolean firstTime = true; | ||
|
|
||
| public JiraIterator(final JiraService service, | ||
| PluginExecutorServiceProvider executorServiceProvider, | ||
| JiraSourceConfig sourceConfig) { | ||
| this.service = service; | ||
| this.crawlerTaskExecutor = executorServiceProvider.get(); | ||
| this.sourceConfig = sourceConfig; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| if (firstTime) { | ||
| log.info("Crawling has been started"); | ||
| itemInfoQueue = service.getJiraEntities(sourceConfig, lastPollTime); | ||
| firstTime = false; | ||
| } | ||
| return !this.itemInfoQueue.isEmpty(); | ||
| } | ||
|
|
||
| @Override | ||
| public ItemInfo next() { | ||
| return this.itemInfoQueue.remove(); | ||
| } | ||
|
|
||
| /** | ||
| * Initialize. | ||
| * | ||
| * @param jiraChangeLogToken the jira change log token | ||
| */ | ||
| public void initialize(Instant jiraChangeLogToken) { | ||
| this.itemInfoQueue = new ConcurrentLinkedQueue<>(); | ||
| this.lastPollTime = jiraChangeLogToken; | ||
| this.firstTime = true; | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.