From 7a2b2ca154f38490c62ab9bb7a441a5f2c5020e1 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 5 Sep 2023 10:45:43 +0100 Subject: [PATCH 1/4] Improvements (#1411) --- .../view/organization-view-members.vue | 3 +-- .../pages/organization-view-page.vue | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/frontend/src/modules/organization/components/view/organization-view-members.vue b/frontend/src/modules/organization/components/view/organization-view-members.vue index 9f007302a0..eb4ae845a0 100644 --- a/frontend/src/modules/organization/components/view/organization-view-members.vue +++ b/frontend/src/modules/organization/components/view/organization-view-members.vue @@ -10,8 +10,7 @@

- Members can take up to two minutes to appear in the - list + No members are currently working in this organization.

diff --git a/frontend/src/modules/organization/pages/organization-view-page.vue b/frontend/src/modules/organization/pages/organization-view-page.vue index 04a71f2095..8bf3aac439 100644 --- a/frontend/src/modules/organization/pages/organization-view-page.vue +++ b/frontend/src/modules/organization/pages/organization-view-page.vue @@ -25,12 +25,25 @@
- + + Date: Tue, 5 Sep 2023 14:02:07 +0300 Subject: [PATCH 2/4] Groups.io integration (#1388) --- backend/.env.dist.local | 2 +- .../helpers/groupsioConnectOrUpdate.ts | 9 + .../integration/helpers/groupsioGetToken.ts | 9 + .../helpers/groupsioVerifyGroup.ts | 9 + backend/src/api/integration/index.ts | 15 + .../integrations/usecases/groupsio/types.ts | 18 + backend/src/services/integrationService.ts | 112 +++++- backend/src/types/webhooks.ts | 1 + frontend/public/icons/crowd-icons.svg | 3 + .../public/images/integrations/groupsio.svg | 197 +++++++++ frontend/scripts/docker-entrypoint.sh | 1 + frontend/src/config.js | 3 + .../components/groupsio-array-input.vue | 83 ++++ .../components/groupsio-connect-drawer.vue | 378 ++++++++++++++++++ .../groupsio/components/groupsio-connect.vue | 32 ++ frontend/src/integrations/groupsio/config.js | 20 + frontend/src/integrations/groupsio/index.js | 3 + .../src/integrations/integrations-config.js | 2 + .../integration/integration-service.js | 44 ++ .../modules/integration/integration-store.js | 45 +++ services/apps/webhook_api/src/main.ts | 2 + .../webhook_api/src/repos/webhooks.repo.ts | 19 + .../apps/webhook_api/src/routes/groupsio.ts | 57 +++ services/libs/integrations/src/helpers.ts | 16 + .../src/integrations/activityTypes.ts | 28 ++ .../groupsio/api/getGroupMember.ts | 28 ++ .../groupsio/api/getGroupMembers.ts | 28 ++ .../groupsio/api/getMessagesFromTopic.ts | 26 ++ .../groupsio/api/getTopicsFromGroup.ts | 28 ++ .../integrations/groupsio/generateStreams.ts | 42 ++ .../src/integrations/groupsio/grid.ts | 18 + .../src/integrations/groupsio/index.ts | 23 ++ .../integrations/groupsio/memberAttributes.ts | 17 + .../src/integrations/groupsio/processData.ts | 131 ++++++ .../integrations/groupsio/processStream.ts | 199 +++++++++ .../groupsio/processWebhookStream.ts | 70 ++++ .../src/integrations/groupsio/types.ts | 363 +++++++++++++++++ .../integrations/src/integrations/index.ts | 4 + services/libs/types/src/enums/platforms.ts | 1 + services/libs/types/src/enums/webhooks.ts | 1 + 40 files changed, 2085 insertions(+), 2 deletions(-) create mode 100644 backend/src/api/integration/helpers/groupsioConnectOrUpdate.ts create mode 100644 backend/src/api/integration/helpers/groupsioGetToken.ts create mode 100644 backend/src/api/integration/helpers/groupsioVerifyGroup.ts create mode 100644 backend/src/serverless/integrations/usecases/groupsio/types.ts create mode 100644 frontend/public/images/integrations/groupsio.svg create mode 100644 frontend/src/integrations/groupsio/components/groupsio-array-input.vue create mode 100644 frontend/src/integrations/groupsio/components/groupsio-connect-drawer.vue create mode 100644 frontend/src/integrations/groupsio/components/groupsio-connect.vue create mode 100644 frontend/src/integrations/groupsio/config.js create mode 100644 frontend/src/integrations/groupsio/index.js create mode 100644 services/apps/webhook_api/src/routes/groupsio.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/api/getGroupMember.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/api/getGroupMembers.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/api/getMessagesFromTopic.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/api/getTopicsFromGroup.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/generateStreams.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/grid.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/index.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/memberAttributes.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/processData.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/processStream.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/processWebhookStream.ts create mode 100644 services/libs/integrations/src/integrations/groupsio/types.ts diff --git a/backend/.env.dist.local b/backend/.env.dist.local index e43cd6c328..aff5e593d0 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -165,4 +165,4 @@ CROWD_WEEKLY_EMAILS_ENABLED="true" CROWD_ANALYTICS_IS_ENABLED= CROWD_ANALYTICS_TENANT_ID= CROWD_ANALYTICS_BASE_URL= -CROWD_ANALYTICS_API_TOKEN= \ No newline at end of file +CROWD_ANALYTICS_API_TOKEN= diff --git a/backend/src/api/integration/helpers/groupsioConnectOrUpdate.ts b/backend/src/api/integration/helpers/groupsioConnectOrUpdate.ts new file mode 100644 index 0000000000..12a2dda7cf --- /dev/null +++ b/backend/src/api/integration/helpers/groupsioConnectOrUpdate.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).groupsioConnectOrUpdate(req.body) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/groupsioGetToken.ts b/backend/src/api/integration/helpers/groupsioGetToken.ts new file mode 100644 index 0000000000..cfedd1ae65 --- /dev/null +++ b/backend/src/api/integration/helpers/groupsioGetToken.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).groupsioGetToken(req.body) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/groupsioVerifyGroup.ts b/backend/src/api/integration/helpers/groupsioVerifyGroup.ts new file mode 100644 index 0000000000..bc7641236f --- /dev/null +++ b/backend/src/api/integration/helpers/groupsioVerifyGroup.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).groupsioVerifyGroup(req.body) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index 4c98042a45..44d1456cec 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -132,6 +132,21 @@ export default (app) => { safeWrap(require('./helpers/hubspotStopSyncOrganization').default), ) + app.post( + '/tenant/:tenantId/groupsio-connect', + safeWrap(require('./helpers/groupsioConnectOrUpdate').default), + ) + + app.post( + '/tenant/:tenantId/groupsio-get-token', + safeWrap(require('./helpers/groupsioGetToken').default), + ) + + app.post( + '/tenant/:tenantId/groupsio-verify-group', + safeWrap(require('./helpers/groupsioVerifyGroup').default), + ) + // if (TWITTER_CONFIG.clientId) { // /** // * Using the passport.authenticate this endpoint forces a diff --git a/backend/src/serverless/integrations/usecases/groupsio/types.ts b/backend/src/serverless/integrations/usecases/groupsio/types.ts new file mode 100644 index 0000000000..60449a3029 --- /dev/null +++ b/backend/src/serverless/integrations/usecases/groupsio/types.ts @@ -0,0 +1,18 @@ +export interface GroupsioIntegrationData { + email: string + token: string + groupNames: GroupName[] +} + +export interface GroupsioGetToken { + email: string + password: string + twoFactorCode?: string +} + +export interface GroupsioVerifyGroup { + groupName: GroupName + cookie: string +} + +export type GroupName = string diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 74bca1a480..5c7556f5de 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -1,7 +1,7 @@ import { createAppAuth } from '@octokit/auth-app' import { request } from '@octokit/request' import moment from 'moment' -import axios from 'axios' +import axios, { AxiosRequestConfig, AxiosResponse } from 'axios' import { PlatformType } from '@crowd/types' import { HubspotFieldMapperFactory, @@ -44,6 +44,11 @@ import OrganizationService from './organizationService' import MemberSyncRemoteRepository from '@/database/repositories/memberSyncRemoteRepository' import OrganizationSyncRemoteRepository from '@/database/repositories/organizationSyncRemoteRepository' import MemberRepository from '@/database/repositories/memberRepository' +import { + GroupsioIntegrationData, + GroupsioGetToken, + GroupsioVerifyGroup, +} from '@/serverless/integrations/usecases/groupsio/types' const discordToken = DISCORD_CONFIG.token || DISCORD_CONFIG.token2 @@ -1403,4 +1408,109 @@ export default class IntegrationService { return integration } + + async groupsioConnectOrUpdate(integrationData: GroupsioIntegrationData) { + const transaction = await SequelizeRepository.createTransaction(this.options) + let integration + + // integration data should have the following fields + // email, token, array of groups + // we shouldn't store password and 2FA token in the database + // user should update them every time thety change something + + try { + this.options.log.info('Creating Groups.io integration!') + integration = await this.createOrUpdate( + { + platform: PlatformType.GROUPSIO, + settings: { + email: integrationData.email, + token: integrationData.token, + groups: integrationData.groupNames, + updateMemberAttributes: true, + }, + status: 'in-progress', + }, + transaction, + ) + + await SequelizeRepository.commitTransaction(transaction) + } catch (err) { + await SequelizeRepository.rollbackTransaction(transaction) + throw err + } + + this.options.log.info( + { tenantId: integration.tenantId }, + 'Sending Groups.io message to int-run-worker!', + ) + const emitter = await getIntegrationRunWorkerEmitter() + await emitter.triggerIntegrationRun( + integration.tenantId, + integration.platform, + integration.id, + true, + ) + + return integration + } + + async groupsioGetToken(data: GroupsioGetToken) { + const config: AxiosRequestConfig = { + method: 'post', + url: 'https://groups.io/api/v1/login', + params: { + email: data.email, + password: data.password, + twofactor: data.twoFactorCode, + }, + headers: { + 'Content-Type': 'application/json', + }, + } + + let response: AxiosResponse + + try { + response = await axios(config) + + // we need to get cookie from the response + + const cookie = response.headers['set-cookie'][0].split(';')[0] + + return { + groupsioCookie: cookie, + } + } catch (err) { + if ('two_factor_required' in response.data) { + throw new Error400(this.options.language, 'errors.groupsio.twoFactorRequired') + } + throw new Error400(this.options.language, 'errors.groupsio.invalidCredentials') + } + } + + async groupsioVerifyGroup(data: GroupsioVerifyGroup) { + const groupName = data.groupName + + const config: AxiosRequestConfig = { + method: 'post', + url: `https://groups.io/api/v1/gettopics?group_name=${encodeURIComponent(groupName)}`, + headers: { + 'Content-Type': 'application/json', + Cookie: data.cookie, + }, + } + + let response: AxiosResponse + + try { + response = await axios(config) + + return { + group: response?.data?.data?.group_id, + } + } catch (err) { + throw new Error400(this.options.language, 'errors.groupsio.invalidGroup') + } + } } diff --git a/backend/src/types/webhooks.ts b/backend/src/types/webhooks.ts index 3fb6ff528b..dfacb7595a 100644 --- a/backend/src/types/webhooks.ts +++ b/backend/src/types/webhooks.ts @@ -10,6 +10,7 @@ export enum WebhookType { GITHUB = 'GITHUB', DISCORD = 'DISCORD', DISCOURSE = 'DISCOURSE', + GROUPSIO = 'GROUPSIO', } export enum DiscordWebsocketEvent { diff --git a/frontend/public/icons/crowd-icons.svg b/frontend/public/icons/crowd-icons.svg index dd5718f88e..b9a09e9712 100644 --- a/frontend/public/icons/crowd-icons.svg +++ b/frontend/public/icons/crowd-icons.svg @@ -102,4 +102,7 @@ + + + diff --git a/frontend/public/images/integrations/groupsio.svg b/frontend/public/images/integrations/groupsio.svg new file mode 100644 index 0000000000..af62003dec --- /dev/null +++ b/frontend/public/images/integrations/groupsio.svg @@ -0,0 +1,197 @@ + + + + + + + + + + + + + diff --git a/frontend/scripts/docker-entrypoint.sh b/frontend/scripts/docker-entrypoint.sh index 6e00ee2e60..cb70699c43 100755 --- a/frontend/scripts/docker-entrypoint.sh +++ b/frontend/scripts/docker-entrypoint.sh @@ -30,6 +30,7 @@ declare -a ENV_VARIABLES=( "VUE_APP_SAMPLE_TENANT_ID" "VUE_APP_SAMPLE_TENANT_TOKEN" "VUE_APP_IS_GIT_ENABLED" + "VUE_APP_IS_GROUPSIO_ENABLED" ) for ENV_VAR in "${ENV_VARIABLES[@]}" diff --git a/frontend/src/config.js b/frontend/src/config.js index e12e927ad9..ee1de75b2b 100644 --- a/frontend/src/config.js +++ b/frontend/src/config.js @@ -57,6 +57,7 @@ const defaultConfig = { token: import.meta.env.VUE_APP_SAMPLE_TENANT_TOKEN, }, isGitEnabled: import.meta.env.VUE_APP_IS_GIT_ENABLED, + isGroupsioEnabled: import.meta.env.VUE_APP_IS_GROUPSIO_ENABLED, }; const composedConfig = { @@ -106,6 +107,7 @@ const composedConfig = { token: 'CROWD_VUE_APP_SAMPLE_TENANT_TOKEN', }, isGitEnabled: 'CROWD_VUE_APP_IS_GIT_ENABLED', + isGroupsioEnabled: 'CROWD_VUE_APP_IS_GROUPSIO_ENABLED', }; const config = defaultConfig.backendUrl @@ -116,5 +118,6 @@ config.isCommunityVersion = config.edition === 'community'; config.hasPremiumModules = !config.isCommunityVersion || config.communityPremium === 'true'; config.isGitIntegrationEnabled = config.isGitEnabled === 'true'; +config.isGroupsioIntegrationEnabled = config.isGroupsioEnabled === 'true'; export default config; diff --git a/frontend/src/integrations/groupsio/components/groupsio-array-input.vue b/frontend/src/integrations/groupsio/components/groupsio-array-input.vue new file mode 100644 index 0000000000..86f58691fc --- /dev/null +++ b/frontend/src/integrations/groupsio/components/groupsio-array-input.vue @@ -0,0 +1,83 @@ + + + + + diff --git a/frontend/src/integrations/groupsio/components/groupsio-connect-drawer.vue b/frontend/src/integrations/groupsio/components/groupsio-connect-drawer.vue new file mode 100644 index 0000000000..c85594609a --- /dev/null +++ b/frontend/src/integrations/groupsio/components/groupsio-connect-drawer.vue @@ -0,0 +1,378 @@ + + + + + diff --git a/frontend/src/integrations/groupsio/components/groupsio-connect.vue b/frontend/src/integrations/groupsio/components/groupsio-connect.vue new file mode 100644 index 0000000000..6de5fc7df6 --- /dev/null +++ b/frontend/src/integrations/groupsio/components/groupsio-connect.vue @@ -0,0 +1,32 @@ + + + + diff --git a/frontend/src/integrations/groupsio/config.js b/frontend/src/integrations/groupsio/config.js new file mode 100644 index 0000000000..3444ea14db --- /dev/null +++ b/frontend/src/integrations/groupsio/config.js @@ -0,0 +1,20 @@ +import config from '@/config'; +import GroupsioConnect from './components/groupsio-connect.vue'; + +export default { + enabled: config.isGroupsioIntegrationEnabled, + hideAsIntegration: !config.isGroupsioIntegrationEnabled, + name: 'Groups.io', + backgroundColor: '#FFFFFF', + borderColor: '#FFFFFF', + description: + 'Connect Groups.io to sync groups and topics activity.', + image: + '/images/integrations/groupsio.svg', + connectComponent: GroupsioConnect, + chartColor: '#111827', + showProfileLink: true, + activityDisplay: { + showLinkToUrl: true, + }, +}; diff --git a/frontend/src/integrations/groupsio/index.js b/frontend/src/integrations/groupsio/index.js new file mode 100644 index 0000000000..e81a18f3ed --- /dev/null +++ b/frontend/src/integrations/groupsio/index.js @@ -0,0 +1,3 @@ +import config from './config'; + +export default config; diff --git a/frontend/src/integrations/integrations-config.js b/frontend/src/integrations/integrations-config.js index 2483760941..b2279eae93 100644 --- a/frontend/src/integrations/integrations-config.js +++ b/frontend/src/integrations/integrations-config.js @@ -15,6 +15,7 @@ import crunchbase from './crunchbase'; import git from './git'; import facebook from './facebook'; import n8n from './n8n'; +import groupsio from './groupsio'; class IntegrationsConfig { get integrations() { @@ -33,6 +34,7 @@ class IntegrationsConfig { git, crunchbase, discourse, + groupsio, hubspot, // make, facebook, diff --git a/frontend/src/modules/integration/integration-service.js b/frontend/src/modules/integration/integration-service.js index 419d5bc9b2..d4ffa008ea 100644 --- a/frontend/src/modules/integration/integration-service.js +++ b/frontend/src/modules/integration/integration-service.js @@ -348,4 +348,48 @@ export class IntegrationService { return response.data.isWebhooksReceived; } + + static async groupsioConnect(email, token, groupNames) { + const tenantId = AuthCurrentTenant.get(); + + const response = await authAxios.post( + `/tenant/${tenantId}/groupsio-connect`, + { + email, + token, + groupNames, + }, + ); + + return response.data; + } + + static async groupsioGetToken(email, password, twoFactorCode = null) { + const tenantId = AuthCurrentTenant.get(); + + const response = await authAxios.post( + `/tenant/${tenantId}/groupsio-get-token`, + { + email, + password, + twoFactorCode, + }, + ); + + return response.data; + } + + static async groupsioVerifyGroup(groupName, cookie) { + const tenantId = AuthCurrentTenant.get(); + + const response = await authAxios.post( + `/tenant/${tenantId}/groupsio-verify-group`, + { + groupName, + cookie, + }, + ); + + return response.data; + } } diff --git a/frontend/src/modules/integration/integration-store.js b/frontend/src/modules/integration/integration-store.js index 11fa998513..d07f87fe98 100644 --- a/frontend/src/modules/integration/integration-store.js +++ b/frontend/src/modules/integration/integration-store.js @@ -546,5 +546,50 @@ export default { commit('CREATE_ERROR'); } }, + + async doGroupsioConnect( + { commit }, + { + email, token, groupNames, isUpdate, + }, + ) { + console.log('doGroupsioConnect', email, token, groupNames, isUpdate); + + try { + commit('CREATE_STARTED'); + + const integration = await IntegrationService.groupsioConnect( + email, + token, + groupNames, + ); + + commit('CREATE_SUCCESS', integration); + + Message.success( + 'The first activities will show up in a couple of seconds.

' + + 'This process might take a few minutes to finish, depending on the amount of data.', + { + title: + ` + groups.io integration ${isUpdate ? 'updated' : 'created'} successfully`, + }, + ); + + router.push({ + name: 'integration', + params: { + id: integration.segmentId, + }, + }); + } catch (error) { + Errors.handle(error); + Message.error( + 'Something went wrong. Please try again later.', + ); + commit('CREATE_ERROR'); + } + }, }, + }; diff --git a/services/apps/webhook_api/src/main.ts b/services/apps/webhook_api/src/main.ts index b7b428966b..d539c0998b 100644 --- a/services/apps/webhook_api/src/main.ts +++ b/services/apps/webhook_api/src/main.ts @@ -8,6 +8,7 @@ import { databaseMiddleware } from './middleware/database' import { errorMiddleware } from './middleware/error' import { sqsMiddleware } from './middleware/sqs' import { installGithubRoutes } from './routes/github' +import { installGroupsIoRoutes } from './routes/groupsio' import cors from 'cors' const log = getServiceLogger() @@ -29,6 +30,7 @@ setImmediate(async () => { // add routes installGithubRoutes(app) + installGroupsIoRoutes(app) app.use(errorMiddleware()) diff --git a/services/apps/webhook_api/src/repos/webhooks.repo.ts b/services/apps/webhook_api/src/repos/webhooks.repo.ts index d7d69b7f5d..db387c70df 100644 --- a/services/apps/webhook_api/src/repos/webhooks.repo.ts +++ b/services/apps/webhook_api/src/repos/webhooks.repo.ts @@ -3,6 +3,7 @@ import { Logger } from '@crowd/logging' import { IDbIntegrationData } from './webhooks.data' import { WebhookState, WebhookType } from '@crowd/types' import { generateUUIDv1 } from '@crowd/common' +import { PlatformType } from '@crowd/types' export class WebhooksRepository extends RepositoryBase { public constructor(dbStore: DbStore, parentLog: Logger) { @@ -53,4 +54,22 @@ export class WebhooksRepository extends RepositoryBase { return id } + + public async findGroupsIoIntegrationByGroupName( + groupName: string, + ): Promise { + const result = await this.db().oneOrNone( + ` + select id, "tenantId", platform from integrations + where platform = $(platform) and "deletedAt" is null + and settings -> 'groups' ? $(groupName) + `, + { + platform: PlatformType.GROUPSIO, + groupName: groupName, + }, + ) + + return result + } } diff --git a/services/apps/webhook_api/src/routes/groupsio.ts b/services/apps/webhook_api/src/routes/groupsio.ts new file mode 100644 index 0000000000..2c704cb3eb --- /dev/null +++ b/services/apps/webhook_api/src/routes/groupsio.ts @@ -0,0 +1,57 @@ +import { asyncWrap } from '@/middleware/error' +import { WebhooksRepository } from '@/repos/webhooks.repo' +import { Error400BadRequest } from '@crowd/common' +import { IntegrationStreamWorkerEmitter } from '@crowd/sqs' +import { WebhookType } from '@crowd/types' +import express from 'express' + +export const installGroupsIoRoutes = async (app: express.Express) => { + let emitter: IntegrationStreamWorkerEmitter + app.post( + '/groupsio', + asyncWrap(async (req, res) => { + const signature = req.headers['x-groupsio-signature'] + const event = req.headers['x-groupsio-action'] + const data = req.body + + // TODO: Validate signature - need to get secret from groups io for Linux + + if (!data?.group?.name) { + throw new Error400BadRequest('Missing group name!') + } + + const repo = new WebhooksRepository(req.dbStore, req.log) + + const integration = await repo.findGroupsIoIntegrationByGroupName(data.group.name) + + if (integration) { + req.log.info({ integrationId: integration.id }, 'Incoming Groups.io Webhook!') + + const result = await repo.createIncomingWebhook( + integration.tenantId, + integration.id, + WebhookType.GROUPSIO, + { + signature, + event, + data, + }, + ) + + if (!emitter) { + emitter = new IntegrationStreamWorkerEmitter(req.sqs, req.log) + await emitter.init() + } + + await emitter.triggerWebhookProcessing(integration.tenantId, integration.platform, result) + + res.sendStatus(204) + } else { + req.log.error({ event }, 'No integration found for incoming Groups.io Webhook!') + res.status(200).send({ + message: 'No integration found for incoming Groups.io Webhook!', + }) + } + }), + ) +} diff --git a/services/libs/integrations/src/helpers.ts b/services/libs/integrations/src/helpers.ts index 6f098ca34b..9205f0409c 100644 --- a/services/libs/integrations/src/helpers.ts +++ b/services/libs/integrations/src/helpers.ts @@ -1,4 +1,5 @@ import crypto from 'crypto' +import * as buffer from 'buffer' /** * Some activities will not have a remote(API) counterparts so they will miss sourceIds. @@ -27,3 +28,18 @@ export function generateSourceIdHash( const data = `${uniqueRemoteId}-${type}-${timestamp}-${platform}` return `gen-${crypto.createHash('md5').update(data).digest('hex')}` } + +export function verifyWebhookSignature( + payload: string, + secret: string, + signatureHeader: string, +): boolean { + const hmac = crypto.createHmac('sha256', secret) + hmac.update(payload) + const expectedSignature = `sha256=${hmac.digest('hex')}` + + return crypto.timingSafeEqual( + buffer.Buffer.from(signatureHeader), + buffer.Buffer.from(expectedSignature), + ) +} diff --git a/services/libs/integrations/src/integrations/activityTypes.ts b/services/libs/integrations/src/integrations/activityTypes.ts index d6045e6de1..3a0efcfe09 100644 --- a/services/libs/integrations/src/integrations/activityTypes.ts +++ b/services/libs/integrations/src/integrations/activityTypes.ts @@ -21,6 +21,8 @@ import { TWITTER_GRID } from './twitter/grid' import { STACKOVERFLOW_GRID } from './stackoverflow/grid' import { DiscourseActivityType } from './discourse/types' import { DISCOURSE_GRID } from './discourse/grid' +import { Groupsio_GRID } from './groupsio/grid' +import { GroupsioActivityType } from './groupsio/types' export const UNKNOWN_ACTIVITY_TYPE_DISPLAY: ActivityTypeDisplayProperties = { default: 'Conducted an activity', @@ -730,4 +732,30 @@ export const DEFAULT_ACTIVITY_TYPE_SETTINGS: DefaultActivityTypes = { isContribution: DISCOURSE_GRID[DiscourseActivityType.LIKE].isContribution, }, }, + [PlatformType.GROUPSIO]: { + [GroupsioActivityType.MEMBER_JOIN]: { + display: { + default: 'Joined {channel}', + short: 'joined', + channel: '{channel}', + }, + isContribution: Groupsio_GRID[GroupsioActivityType.MEMBER_JOIN].isContribution, + }, + [GroupsioActivityType.MESSAGE]: { + display: { + default: 'Sent a message in {channel}', + short: 'sent a message', + channel: '{channel}', + }, + isContribution: Groupsio_GRID[GroupsioActivityType.MESSAGE].isContribution, + }, + [GroupsioActivityType.MEMBER_LEAVE]: { + display: { + default: 'Left {channel}', + short: 'left', + channel: '{channel}', + }, + isContribution: Groupsio_GRID[GroupsioActivityType.MEMBER_LEAVE].isContribution, + }, + }, } diff --git a/services/libs/integrations/src/integrations/groupsio/api/getGroupMember.ts b/services/libs/integrations/src/integrations/groupsio/api/getGroupMember.ts new file mode 100644 index 0000000000..eda8f33c0d --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/api/getGroupMember.ts @@ -0,0 +1,28 @@ +import axios, { AxiosRequestConfig } from 'axios' +import { IProcessStreamContext } from '@/types' +import { GroupName } from '../types' + +export const getGroupMember = async ( + memberInfoId: string, + groupName: GroupName, + cookie: string, + ctx: IProcessStreamContext, +) => { + const config: AxiosRequestConfig = { + method: 'get', + url: `https://groups.io/api/v1/getmember?member_info_id=${encodeURI( + memberInfoId, + )}&group_name=${encodeURI(groupName)}`, + headers: { + Cookie: cookie, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + ctx.log.error(err, { memberInfoId }, 'Error fetching member by memberInfoId!') + throw err + } +} diff --git a/services/libs/integrations/src/integrations/groupsio/api/getGroupMembers.ts b/services/libs/integrations/src/integrations/groupsio/api/getGroupMembers.ts new file mode 100644 index 0000000000..93dbacaec2 --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/api/getGroupMembers.ts @@ -0,0 +1,28 @@ +import axios, { AxiosRequestConfig } from 'axios' +import { IProcessStreamContext } from '@/types' +import { GroupName } from '../types' + +export const getGroupMembers = async ( + groupName: GroupName, + cookie: string, + ctx: IProcessStreamContext, + page = null, +) => { + const config: AxiosRequestConfig = { + method: 'get', + url: + `https://groups.io/api/v1/getmembers?group_name=${encodeURIComponent(groupName)}` + + (page ? `&page_token=${page}` : ''), + headers: { + Cookie: cookie, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + ctx.log.error(err, { groupName }, 'Error fetching members from group!') + throw err + } +} diff --git a/services/libs/integrations/src/integrations/groupsio/api/getMessagesFromTopic.ts b/services/libs/integrations/src/integrations/groupsio/api/getMessagesFromTopic.ts new file mode 100644 index 0000000000..936f16af31 --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/api/getMessagesFromTopic.ts @@ -0,0 +1,26 @@ +import axios, { AxiosRequestConfig } from 'axios' +import { IProcessStreamContext } from '@/types' + +export const getMessagesFromTopic = async ( + topicId: string, + cookie: string, + ctx: IProcessStreamContext, + page: string = null, +) => { + const config: AxiosRequestConfig = { + method: 'get', + url: + `https://groups.io/api/v1/gettopic?topic_id=${topicId}` + (page ? `&page_token=${page}` : ''), + headers: { + Cookie: cookie, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + ctx.log.error(err, { topic_id: topicId }, 'Error fetching messags from topic_id!') + throw err + } +} diff --git a/services/libs/integrations/src/integrations/groupsio/api/getTopicsFromGroup.ts b/services/libs/integrations/src/integrations/groupsio/api/getTopicsFromGroup.ts new file mode 100644 index 0000000000..8839b88b8f --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/api/getTopicsFromGroup.ts @@ -0,0 +1,28 @@ +import axios, { AxiosRequestConfig } from 'axios' +import { IProcessStreamContext } from '@/types' +import { GroupName } from '../types' + +export const getTopicsFromGroup = async ( + groupName: GroupName, + cookie: string, + ctx: IProcessStreamContext, + page: string = null, +) => { + const config: AxiosRequestConfig = { + method: 'get', + url: + `https://groups.io/api/v1/gettopics?group_name=${encodeURIComponent(groupName)}` + + (page ? `&page_token=${page}` : ''), + headers: { + Cookie: cookie, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + ctx.log.error(err, { groupName }, 'Error fetching topics from group!') + throw err + } +} diff --git a/services/libs/integrations/src/integrations/groupsio/generateStreams.ts b/services/libs/integrations/src/integrations/groupsio/generateStreams.ts new file mode 100644 index 0000000000..b433415c3f --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/generateStreams.ts @@ -0,0 +1,42 @@ +// generateStreams.ts content +import { GenerateStreamsHandler } from '../../types' +import { + GroupsioIntegrationSettings, + GroupsioStreamType, + GroupsioGroupMembersStreamMetadata, +} from './types' + +const handler: GenerateStreamsHandler = async (ctx) => { + const settings = ctx.integration.settings as GroupsioIntegrationSettings + + const groups = settings.groups + const token = settings.token + const email = settings.email + + if (!groups || groups.length === 0) { + await ctx.abortRunWithError('No groups specified!') + } + + if (!token) { + await ctx.abortRunWithError('No token specified!') + } + + if (!email) { + await ctx.abortRunWithError('No email specified!') + } + + for (const group of groups) { + // here we start parsing group members - very important to do this first + // because we need to know who the members are before we can start parsing + // messages don't have enough information to create members + await ctx.publishStream( + `${GroupsioStreamType.GROUP_MEMBERS}:${group}`, + { + group, + page: null, + }, + ) + } +} + +export default handler diff --git a/services/libs/integrations/src/integrations/groupsio/grid.ts b/services/libs/integrations/src/integrations/groupsio/grid.ts new file mode 100644 index 0000000000..b798230d0d --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/grid.ts @@ -0,0 +1,18 @@ +// grid.ts content +import { IActivityScoringGrid } from '@crowd/types' +import { GroupsioActivityType } from './types' + +export const Groupsio_GRID: Record = { + [GroupsioActivityType.MEMBER_JOIN]: { + score: 2, + isContribution: false, + }, + [GroupsioActivityType.MESSAGE]: { + score: 6, + isContribution: true, + }, + [GroupsioActivityType.MEMBER_LEAVE]: { + score: -2, + isContribution: false, + }, +} diff --git a/services/libs/integrations/src/integrations/groupsio/index.ts b/services/libs/integrations/src/integrations/groupsio/index.ts new file mode 100644 index 0000000000..edf3563469 --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/index.ts @@ -0,0 +1,23 @@ +// index.ts content +import { IIntegrationDescriptor } from '../../types' +import generateStreams from './generateStreams' +import { GROUPSIO_MEMBER_ATTRIBUTES } from './memberAttributes' +import processStream from './processStream' +import processData from './processData' +import processWebhookStream from './processWebhookStream' +import { PlatformType } from '@crowd/types' + +const descriptor: IIntegrationDescriptor = { + type: PlatformType.GROUPSIO, + memberAttributes: GROUPSIO_MEMBER_ATTRIBUTES, + generateStreams, + processStream, + processData, + processWebhookStream, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + postProcess: (settings: any) => { + return settings + }, +} + +export default descriptor diff --git a/services/libs/integrations/src/integrations/groupsio/memberAttributes.ts b/services/libs/integrations/src/integrations/groupsio/memberAttributes.ts new file mode 100644 index 0000000000..21d2e80e38 --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/memberAttributes.ts @@ -0,0 +1,17 @@ +// memberAttributes.ts content +import { + IMemberAttribute, + MemberAttributeName, + MemberAttributeType, + MemberAttributes, +} from '@crowd/types' + +export const GROUPSIO_MEMBER_ATTRIBUTES: IMemberAttribute[] = [ + { + name: MemberAttributes[MemberAttributeName.SOURCE_ID].name, + label: MemberAttributes[MemberAttributeName.SOURCE_ID].label, + type: MemberAttributeType.STRING, + canDelete: false, + show: false, + }, +] diff --git a/services/libs/integrations/src/integrations/groupsio/processData.ts b/services/libs/integrations/src/integrations/groupsio/processData.ts new file mode 100644 index 0000000000..1a9c22963f --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/processData.ts @@ -0,0 +1,131 @@ +// processData.ts content +import { ProcessDataHandler } from '../../types' +import { Groupsio_GRID } from './grid' +import { + GroupsioPublishData, + GroupsioPublishDataType, + GroupsioMessageData, + MemberInfo, + GroupsioMemberJoinData, + GroupsioMemberLeftData, + GroupsioActivityType, +} from './types' +import { IActivityData, IMemberData, PlatformType } from '@crowd/types' +import sanitizeHtml from 'sanitize-html' + +const processMemberJoin: ProcessDataHandler = async (ctx) => { + const metaData = ctx.data as GroupsioPublishData + const data = metaData.data + const memberData = data.member as MemberInfo + + const member: IMemberData = { + displayName: memberData.full_name, + emails: [memberData.email], + identities: [ + { + sourceId: memberData.user_id.toString(), + platform: PlatformType.GROUPSIO, + username: memberData.email, + }, + ], + } + + const activity: IActivityData = { + type: GroupsioActivityType.MEMBER_JOIN, + member, + channel: data.group, + timestamp: data.joinedAt, + sourceId: `join-${memberData.user_id}-${memberData.group_id}-${data.joinedAt}`, + score: Groupsio_GRID[GroupsioActivityType.MEMBER_JOIN].score, + isContribution: Groupsio_GRID[GroupsioActivityType.MEMBER_JOIN].isContribution, + } + + await ctx.publishActivity(activity) +} + +const processMessage: ProcessDataHandler = async (ctx) => { + const data = ctx.data as GroupsioPublishData + const messageData = data.data + const memberData = messageData.member + + const member: IMemberData = { + displayName: memberData.full_name, + emails: [memberData.email], + identities: [ + { + sourceId: memberData.user_id.toString(), + platform: PlatformType.GROUPSIO, + username: memberData.email, + }, + ], + } + + const activity: IActivityData = { + type: GroupsioActivityType.MESSAGE, + member, + channel: messageData.group, + timestamp: new Date(messageData.message.created).toISOString(), + sourceId: `${messageData.message.id}`, + score: Groupsio_GRID[GroupsioActivityType.MESSAGE].score, + body: sanitizeHtml(messageData.message.body), + title: sanitizeHtml(messageData.topic.subject), + ...(messageData.sourceParentId && { + sourceParentId: messageData.sourceParentId, + }), + isContribution: Groupsio_GRID[GroupsioActivityType.MESSAGE].isContribution, + } + + await ctx.publishActivity(activity) +} + +const processMemberLeft: ProcessDataHandler = async (ctx) => { + const metaData = ctx.data as GroupsioPublishData + const data = metaData.data + const memberData = data.member as MemberInfo + + const member: IMemberData = { + displayName: memberData.full_name, + emails: [memberData.email], + identities: [ + { + sourceId: memberData.user_id.toString(), + platform: PlatformType.GROUPSIO, + username: memberData.email, + }, + ], + } + + const activity: IActivityData = { + type: GroupsioActivityType.MEMBER_LEAVE, + member, + channel: data.group, + timestamp: data.leftAt, + sourceId: `left-${memberData.user_id}-${memberData.group_id}-${data.leftAt}`, + score: Groupsio_GRID[GroupsioActivityType.MEMBER_LEAVE].score, + isContribution: Groupsio_GRID[GroupsioActivityType.MEMBER_LEAVE].isContribution, + } + + await ctx.publishActivity(activity) +} + +const handler: ProcessDataHandler = async (ctx) => { + const data = ctx.data as GroupsioPublishData + + const type = data.type + + switch (type) { + case GroupsioPublishDataType.MEMBER_JOIN: + await processMemberJoin(ctx) + break + case GroupsioPublishDataType.MESSAGE: + await processMessage(ctx) + break + case GroupsioPublishDataType.MEMBER_LEFT: + await processMemberLeft(ctx) + break + default: + await ctx.abortRunWithError(`Unknown publish data type: ${type}`) + } +} + +export default handler diff --git a/services/libs/integrations/src/integrations/groupsio/processStream.ts b/services/libs/integrations/src/integrations/groupsio/processStream.ts new file mode 100644 index 0000000000..baadd4a9ad --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/processStream.ts @@ -0,0 +1,199 @@ +// processStream.ts content +import { IProcessStreamContext, ProcessStreamHandler } from '../../types' +import { + GroupsioStreamType, + GroupsioGroupStreamMetadata, + GroupsioGroupMembersStreamMetadata, + GroupsioIntegrationSettings, + GroupsioTopicStreamMetadata, + GroupsioPublishData, + GroupsioPublishDataType, + MemberInfo, + ListMembers, + ListMessages, + ListTopics, + GroupsioMessageData, + GroupsioMemberJoinData, +} from './types' +import { getTopicsFromGroup } from './api/getTopicsFromGroup' +import { getMessagesFromTopic } from './api/getMessagesFromTopic' +import { getGroupMembers } from './api/getGroupMembers' + +const cacheMember = async (ctx: IProcessStreamContext, member: MemberInfo): Promise => { + const cacheKey = `${GroupsioStreamType.GROUP_MEMBERS}:${member.user_id}` + const cache = ctx.cache + + // cache for 7 days + await cache.set(cacheKey, JSON.stringify(member), 60 * 60 * 24 * 7) +} + +const getMemberFromCache = async ( + ctx: IProcessStreamContext, + userId: string, +): Promise => { + const cacheKey = `${GroupsioStreamType.GROUP_MEMBERS}:${userId}` + const cache = ctx.cache + + const cachedMember = await cache.get(cacheKey) + + if (!cachedMember) { + return undefined + } + + return JSON.parse(cachedMember) +} + +const processGroupStream: ProcessStreamHandler = async (ctx) => { + const data = ctx.stream.data as GroupsioGroupStreamMetadata + const settings = ctx.integration.settings as GroupsioIntegrationSettings + + const response = (await getTopicsFromGroup( + data.group, + settings.token, + ctx, + data.page, + )) as ListTopics + + // processing next page stream + if (response?.next_page_token) { + await ctx.publishStream( + `${GroupsioStreamType.GROUP}-${data.group}-${response.next_page_token}`, + { + group: data.group, + page: response.next_page_token.toString(), + }, + ) + } + + // publishing topic streams + for (const topic of response.data) { + await ctx.publishStream( + `${GroupsioStreamType.TOPIC}:${topic.id}`, + { + group: data.group, + topic, + page: null, + }, + ) + } +} + +const processTopicStream: ProcessStreamHandler = async (ctx) => { + const data = ctx.stream.data as GroupsioTopicStreamMetadata + const settings = ctx.integration.settings as GroupsioIntegrationSettings + + const response = (await getMessagesFromTopic( + data.topic.id.toString(), + settings.token, + ctx, + data.page, + )) as ListMessages + + // processing next page stream + if (response?.next_page_token) { + await ctx.publishStream( + `${GroupsioStreamType.TOPIC}-${data.topic.id}-${response.next_page_token}`, + { + group: data.group, + topic: data.topic, + page: response.next_page_token.toString(), + }, + ) + } + + // publishing messages + for (let i = 0; i < response.data.length; i++) { + const message = response.data[i] + const userId = message.user_id.toString() + // getting member from cache + // it should be there already because we process members first + const member = await getMemberFromCache(ctx, userId) + + if (!member) { + await ctx.abortRunWithError( + `Member ${userId} not found in cache, we can't process the message!`, + ) + } + + await ctx.publishData>({ + type: GroupsioPublishDataType.MESSAGE, + data: { + message, + group: data.group, + topic: data.topic, + member, + sourceParentId: i > 0 ? response.data[i - 1].id.toString() : null, + }, + }) + } +} + +const processGroupMembersStream: ProcessStreamHandler = async (ctx) => { + const data = ctx.stream.data as GroupsioGroupMembersStreamMetadata + const settings = ctx.integration.settings as GroupsioIntegrationSettings + + const response = (await getGroupMembers( + data.group, + settings.token, + ctx, + data.page, + )) as ListMembers + + // publish members + for (const member of response.data) { + // caching member + await cacheMember(ctx, member) + // publishing member + await ctx.publishData>({ + type: GroupsioPublishDataType.MEMBER_JOIN, + data: { + member, + group: data.group, + joinedAt: new Date(member.created).toISOString(), + }, + }) + } + + // processing next page stream + if (response?.next_page_token) { + await ctx.publishStream( + `${GroupsioStreamType.GROUP_MEMBERS}-${data.group}-${response.next_page_token}`, + { + group: data.group, + page: response.next_page_token.toString(), + }, + ) + } else { + // this is the last page, so we can publish the group streams + await ctx.publishStream( + `${GroupsioStreamType.GROUP}:${data.group}`, + { + group: data.group, + page: null, + }, + ) + } +} + +const handler: ProcessStreamHandler = async (ctx) => { + const streamIdentifier = ctx.stream.identifier + + const streamType = streamIdentifier.split(':')[0] + + switch (streamType) { + case GroupsioStreamType.GROUP: + await processGroupStream(ctx) + break + case GroupsioStreamType.TOPIC: + await processTopicStream(ctx) + break + case GroupsioStreamType.GROUP_MEMBERS: + await processGroupMembersStream(ctx) + break + default: + await ctx.abortRunWithError(`Unknown stream type: ${streamType}`) + break + } +} + +export default handler diff --git a/services/libs/integrations/src/integrations/groupsio/processWebhookStream.ts b/services/libs/integrations/src/integrations/groupsio/processWebhookStream.ts new file mode 100644 index 0000000000..a7a4e81fba --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/processWebhookStream.ts @@ -0,0 +1,70 @@ +import { ProcessWebhookStreamHandler } from '@/types' +import { + GroupsioWebhookEventType, + GroupsioWebhookPayload, + GroupsioWebhookJoinPayload, + GroupsioPublishDataType, + GroupsioPublishData, + // GroupsioMessageData, + GroupsioMemberLeftData, + GroupsioMemberJoinData, +} from './types' + +const processWebhookJoin: ProcessWebhookStreamHandler = async (ctx) => { + const data = ctx.stream.data as GroupsioWebhookPayload + const payload = data.data + + await ctx.publishData>({ + type: GroupsioPublishDataType.MEMBER_JOIN, + data: { + member: payload.member_info, + group: payload.group.name, + joinedAt: new Date(payload.created).toISOString(), + }, + }) +} + +const processWebhookLeft: ProcessWebhookStreamHandler = async (ctx) => { + const data = ctx.stream.data as GroupsioWebhookPayload + const payload = data.data + + await ctx.publishData>({ + type: GroupsioPublishDataType.MEMBER_LEFT, + data: { + member: payload.member_info, + group: payload.group.name, + leftAt: new Date(payload.created).toISOString(), + }, + }) +} + +// const processWebhookSentMessageAccepted: ProcessWebhookStreamHandler = async (ctx) => {} + +const handler: ProcessWebhookStreamHandler = async (ctx) => { + const { event } = ctx.stream.data as GroupsioWebhookPayload + + //verifyWebhookSignature(data as string, data as string, ctx) + + switch (event) { + case GroupsioWebhookEventType.JOINED: { + await processWebhookJoin(ctx) + break + } + + // case GroupsioWebhookEventType.SENT_MESSAGE_ACCEPTED: { + // await processWebhookSentMessageAccepted(ctx) + // break + // } + + case GroupsioWebhookEventType.LEFT: { + await processWebhookLeft(ctx) + break + } + + default: { + ctx.log.warn({ event }, `Unsupported Groupsio webhook type, skipping it`) + } + } +} + +export default handler diff --git a/services/libs/integrations/src/integrations/groupsio/types.ts b/services/libs/integrations/src/integrations/groupsio/types.ts new file mode 100644 index 0000000000..f6fc67bc0e --- /dev/null +++ b/services/libs/integrations/src/integrations/groupsio/types.ts @@ -0,0 +1,363 @@ +// types.ts content +export enum GroupsioActivityType { + MEMBER_JOIN = 'member_join', + MESSAGE = 'message', + MEMBER_LEAVE = 'member_leave', +} + +export type GroupName = string + +export enum GroupsioStreamType { + GROUP = 'group', + GROUP_MEMBERS = 'group_members', + TOPIC = 'topic', +} + +export enum GroupsioPublishDataType { + MESSAGE = 'message', + MEMBER_JOIN = 'member_join', + MEMBER_LEFT = 'member_left', +} + +export interface GroupsioMessageData { + group: GroupName + topic: Topic + message: Message + member: MemberInfo + sourceParentId: string | null +} + +export interface GroupsioMemberJoinData { + group: GroupName + member: MemberInfo + joinedAt: string +} + +export interface GroupsioMemberLeftData { + group: GroupName + member: MemberInfo + leftAt: string +} + +export interface GroupsioGroupStreamMetadata { + group: GroupName + page: string | null +} + +export interface GroupsioGroupMembersStreamMetadata { + group: GroupName + page: string | null +} + +export interface GroupsioMemberStreamMetadata { + group: GroupName + memberId: string +} + +export interface GroupsioTopicStreamMetadata { + group: GroupName + topic: Topic + page: string | null +} + +export interface GroupsioPublishData { + type: GroupsioPublishDataType + data: T +} + +export interface GroupsioIntegrationSettings { + email: string + token: string + groups: GroupName[] +} + +export interface Topic { + id: number + object: 'topic' + created: string + updated: string + group_id: number + group_subject_tag: string + subject: string + summary: string + name: string + profile_photo_url: string + num_messages: number + is_sticky: boolean + is_moderated: boolean + is_closed: boolean + has_attachments: boolean + reply_to: string + most_recent_message: string + hashtags: null | string[] +} + +export interface ListBase { + object: 'list' + total_count: number + start_item: number + end_item: number + has_more: boolean + next_page_token: number + sort_field: string + second_order: string + query: string + sort_dir: 'asc' | 'desc' +} + +export interface ListTopics extends ListBase { + data: Topic[] +} + +export interface ListMessages extends ListBase { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + group_perms?: any + // eslint-disable-next-line @typescript-eslint/no-explicit-any + group?: any + cover_photo_url?: string + // eslint-disable-next-line @typescript-eslint/no-explicit-any + sub_data?: any + topic: Topic + data: Message[] +} + +export interface Message { + id: number + object: string + created: string + updated: string + user_id: number + group_id: number + topic_id: number + body: string + quoted: string + remainder: string + snippet: string + subject: string + subject_with_tags: string + name: string + profile_photo_url: string + is_plain_text: boolean + msg_num: number + is_reply: boolean + has_liked: boolean + num_likes: number + is_closed: boolean + is_moderated: boolean + reply_to: string + can_repost: boolean + hashtags: Hashtag[] + poll_id: number + attachments: Attachment[] +} + +export interface Hashtag { + id: number + object: string + created: string + group_id: number + name: string + mods_only_post: boolean + mods_only_replies: boolean + no_email: boolean + moderated: boolean + special: boolean + replies_unmoderated: boolean + locked: boolean + until: string + close_instead_of_delete: boolean + description: string + color_name: string + color_hex: string + reply_to: string + topic_count: number + last_message_date: string + // eslint-disable-next-line @typescript-eslint/no-explicit-any + muted: null | any + // eslint-disable-next-line @typescript-eslint/no-explicit-any + followed: null | any +} + +export interface Attachment { + id: number + media_type: string + download_url: string + image_thumbnail_url: string + filename: string +} + +interface Perms { + object: string + archives_visible: boolean + polls_visible: boolean + members_visible: boolean + chat_visible: boolean + calendar_visible: boolean + files_visible: boolean + database_visible: boolean + photos_visible: boolean + wiki_visible: boolean + member_directory_visible: boolean + hashtags_visible: boolean + guidelines_visible: boolean + subgroups_visible: boolean + open_donations_visible: boolean + sponsor_visible: boolean + manage_subgroups: boolean + delete_group: boolean + download_archives: boolean + download_entire_group: boolean + download_members: boolean + view_activity: boolean + create_hashtags: boolean + manage_hashtags: boolean + manage_integrations: boolean + manage_group_settings: boolean + make_moderator: boolean + manage_member_subscription_options: boolean + manage_pending_members: boolean + remove_members: boolean + ban_members: boolean + manage_group_billing: boolean + manage_group_payments: boolean + edit_archives: boolean + manage_pending_messages: boolean + invite_members: boolean + can_post: boolean + can_vote: boolean + manage_polls: boolean + manage_photos: boolean + manage_members: boolean + manage_calendar: boolean + manage_chats: boolean + view_member_directory: boolean + manage_files: boolean + manage_wiki: boolean + manage_subscription: boolean + public_page: boolean + sub_page: boolean + mod_page: boolean +} + +interface ExtraMemberData { + col_id: number + col_type: string + text?: string + checked?: boolean + date?: string + time?: string + street_address1?: string + street_address2?: string + city?: string + state?: string + zip?: string + country?: string + title?: string + url?: string + desc?: string + image_name?: string +} + +export interface MemberInfo { + id: number + object: string + created: string + updated: string + user_id: number + group_id: number + group_name: string + status: string + post_status: string + email_delivery: string + message_selection: string + auto_follow_replies: boolean + max_attachment_size: string + approved_posts: number + mod_status: string + pending_msg_notify: string + pending_sub_notify: string + sub_notify: string + storage_notify: string + sub_group_notify: string + message_report_notify: string + account_notify: string + mod_permissions: string + owner_msg_notify: string + chat_notify: string + photo_notify: string + file_notify: string + wiki_notify: string + database_notify: string + email: string + user_status: string + user_name: string + timezone: string + full_name: string + about_me: string + location: string + website: string + profile_privacy: string + dont_munge_message_id: boolean + use_signature: boolean + use_signature_email: boolean + signature: string + color: string + cover_photo_url: string + icon_url: string + nice_group_name: string + subs_count: number + most_recent_message: string + perms: Perms + extra_member_data: ExtraMemberData[] +} + +export interface ListMembers { + object: string + total_count: number + start_item: number + end_item: number + has_more: boolean + next_page_token: number + sort_field: string + second_order: string + query: string + sort_dir: string + data: MemberInfo[] +} + +export enum GroupsioWebhookEventType { + JOINED = 'joined', + SENT_MESSAGE_ACCEPTED = 'sent_message_accepted', + LEFT = 'left', +} + +export interface GroupsioWebhookPayload { + event: GroupsioWebhookEventType + data: T + signature: string +} + +export interface Group { + id: number + object: string + created: string + updated: string + type: string + title: string + name: string + nice_group_name: string + alias: string + desc: string + plain_desc: string +} + +export interface GroupsioWebhookJoinPayload { + id: number + object: string + created: string + webhook_id: number + action: string + via: string + group: Group + member_info: MemberInfo +} diff --git a/services/libs/integrations/src/integrations/index.ts b/services/libs/integrations/src/integrations/index.ts index 822d6a51f1..bccac5c87b 100644 --- a/services/libs/integrations/src/integrations/index.ts +++ b/services/libs/integrations/src/integrations/index.ts @@ -49,3 +49,7 @@ export * from './stackoverflow/memberAttributes' export * from './twitter/grid' export * from './twitter/types' export * from './twitter/memberAttributes' + +export * from './groupsio/grid' +export * from './groupsio/types' +export * from './groupsio/memberAttributes' diff --git a/services/libs/types/src/enums/platforms.ts b/services/libs/types/src/enums/platforms.ts index 418e7d6be4..e588c39e41 100644 --- a/services/libs/types/src/enums/platforms.ts +++ b/services/libs/types/src/enums/platforms.ts @@ -19,6 +19,7 @@ export enum PlatformType { GIT = 'git', CRUNCHBASE = 'crunchbase', HUBSPOT = 'hubspot', + GROUPSIO = 'groupsio', OTHER = 'other', } diff --git a/services/libs/types/src/enums/webhooks.ts b/services/libs/types/src/enums/webhooks.ts index 498fa5fa3d..9284eaba3c 100644 --- a/services/libs/types/src/enums/webhooks.ts +++ b/services/libs/types/src/enums/webhooks.ts @@ -8,6 +8,7 @@ export enum WebhookType { GITHUB = 'GITHUB', DISCORD = 'DISCORD', DISCOURSE = 'DISCOURSE', + GROUPSIO = 'GROUPSIO', CROWD_GENERATED = 'CROWD_GENERATED', } From 6d3597001266dc63cfc46b74ddb5c16a6b15e1fc Mon Sep 17 00:00:00 2001 From: Igor Kotua <36304232+garrrikkotua@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:02:27 +0300 Subject: [PATCH 3/4] Remove 1 when getting db connection (#1421) --- .../src/bin/process-data-for-tenant.ts | 2 +- services/apps/integration_data_worker/src/bin/process-data.ts | 2 +- services/apps/integration_run_worker/src/bin/continue-run.ts | 4 +++- .../integration_run_worker/src/bin/onboard-integration.ts | 2 +- services/apps/integration_run_worker/src/bin/process-repo.ts | 2 +- .../integration_run_worker/src/bin/trigger-all-onboardings.ts | 2 +- .../apps/integration_stream_worker/src/bin/process-stream.ts | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/services/apps/integration_data_worker/src/bin/process-data-for-tenant.ts b/services/apps/integration_data_worker/src/bin/process-data-for-tenant.ts index e5da4e7d13..4d8807159e 100644 --- a/services/apps/integration_data_worker/src/bin/process-data-for-tenant.ts +++ b/services/apps/integration_data_worker/src/bin/process-data-for-tenant.ts @@ -21,7 +21,7 @@ setImmediate(async () => { const emitter = new IntegrationDataWorkerEmitter(sqsClient, log) await emitter.init() - const dbConnection = await getDbConnection(DB_CONFIG(), 1) + const dbConnection = await getDbConnection(DB_CONFIG()) const store = new DbStore(log, dbConnection) const repo = new IntegrationDataRepository(store, log) diff --git a/services/apps/integration_data_worker/src/bin/process-data.ts b/services/apps/integration_data_worker/src/bin/process-data.ts index a28651fce0..ae29f86d5e 100644 --- a/services/apps/integration_data_worker/src/bin/process-data.ts +++ b/services/apps/integration_data_worker/src/bin/process-data.ts @@ -21,7 +21,7 @@ setImmediate(async () => { const emitter = new IntegrationDataWorkerEmitter(sqsClient, log) await emitter.init() - const dbConnection = await getDbConnection(DB_CONFIG(), 1) + const dbConnection = await getDbConnection(DB_CONFIG()) const store = new DbStore(log, dbConnection) const repo = new IntegrationDataRepository(store, log) diff --git a/services/apps/integration_run_worker/src/bin/continue-run.ts b/services/apps/integration_run_worker/src/bin/continue-run.ts index 71d5a82946..c9dd72326f 100644 --- a/services/apps/integration_run_worker/src/bin/continue-run.ts +++ b/services/apps/integration_run_worker/src/bin/continue-run.ts @@ -16,7 +16,7 @@ setImmediate(async () => { const emitter = new IntegrationRunWorkerEmitter(sqsClient, log) await emitter.init() - const dbConnection = await getDbConnection(DB_CONFIG(), 1) + const dbConnection = await getDbConnection(DB_CONFIG()) const store = new DbStore(log, dbConnection) const repo = new IntegrationRunRepository(store, log) @@ -24,6 +24,8 @@ setImmediate(async () => { const run = await repo.findIntegrationRunById(runId) if (run) { + log.info({ run }, 'Found run!') + if (run.state != IntegrationRunState.PENDING) { log.warn(`Integration run is not pending, setting to pending!`) diff --git a/services/apps/integration_run_worker/src/bin/onboard-integration.ts b/services/apps/integration_run_worker/src/bin/onboard-integration.ts index 9dc8a7b986..33d068f9ae 100644 --- a/services/apps/integration_run_worker/src/bin/onboard-integration.ts +++ b/services/apps/integration_run_worker/src/bin/onboard-integration.ts @@ -16,7 +16,7 @@ setImmediate(async () => { const emitter = new IntegrationRunWorkerEmitter(sqsClient, log) await emitter.init() - const dbConnection = await getDbConnection(DB_CONFIG(), 1) + const dbConnection = await getDbConnection(DB_CONFIG()) const store = new DbStore(log, dbConnection) const repo = new IntegrationRunRepository(store, log) diff --git a/services/apps/integration_run_worker/src/bin/process-repo.ts b/services/apps/integration_run_worker/src/bin/process-repo.ts index 3b474503d1..66f2581dd2 100644 --- a/services/apps/integration_run_worker/src/bin/process-repo.ts +++ b/services/apps/integration_run_worker/src/bin/process-repo.ts @@ -68,7 +68,7 @@ setImmediate(async () => { const emitter = new IntegrationRunWorkerEmitter(sqsClient, log) await emitter.init() - const dbConnection = await getDbConnection(DB_CONFIG(), 1) + const dbConnection = await getDbConnection(DB_CONFIG()) const store = new DbStore(log, dbConnection) const repo = new IntegrationRunRepository(store, log) diff --git a/services/apps/integration_run_worker/src/bin/trigger-all-onboardings.ts b/services/apps/integration_run_worker/src/bin/trigger-all-onboardings.ts index b83f30c3bb..ab83cd4e6f 100644 --- a/services/apps/integration_run_worker/src/bin/trigger-all-onboardings.ts +++ b/services/apps/integration_run_worker/src/bin/trigger-all-onboardings.ts @@ -13,7 +13,7 @@ setImmediate(async () => { const emitter = new IntegrationRunWorkerEmitter(sqsClient, log) await emitter.init() - const dbConnection = await getDbConnection(DB_CONFIG(), 1) + const dbConnection = await getDbConnection(DB_CONFIG()) const store = new DbStore(log, dbConnection) const repo = new IntegrationRunRepository(store, log) diff --git a/services/apps/integration_stream_worker/src/bin/process-stream.ts b/services/apps/integration_stream_worker/src/bin/process-stream.ts index b9fb064482..cc809c52df 100644 --- a/services/apps/integration_stream_worker/src/bin/process-stream.ts +++ b/services/apps/integration_stream_worker/src/bin/process-stream.ts @@ -21,7 +21,7 @@ setImmediate(async () => { const emitter = new IntegrationStreamWorkerEmitter(sqsClient, log) await emitter.init() - const dbConnection = await getDbConnection(DB_CONFIG(), 1) + const dbConnection = await getDbConnection(DB_CONFIG()) const store = new DbStore(log, dbConnection) const repo = new IntegrationStreamRepository(store, log) From ecb00e9c71ecd4ddbe4dff5905b59b5a87a4bcec Mon Sep 17 00:00:00 2001 From: anilb Date: Tue, 5 Sep 2023 13:05:49 +0200 Subject: [PATCH 4/4] Segments missing activity types for the display service (#1422) --- backend/src/database/repositories/segmentRepository.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/database/repositories/segmentRepository.ts b/backend/src/database/repositories/segmentRepository.ts index cc50bba250..2aa66dbccf 100644 --- a/backend/src/database/repositories/segmentRepository.ts +++ b/backend/src/database/repositories/segmentRepository.ts @@ -671,8 +671,12 @@ class SegmentRepository extends RepositoryBase< } }) - // TODO: Add member count to segments after implementing member relations - return { count, rows, limit: criteria.limit, offset: criteria.offset } + return { + count, + rows: rows.map((sr) => SegmentRepository.populateRelations(sr)), + limit: criteria.limit, + offset: criteria.offset, + } } private async queryIntegrationsForSubprojects(subprojects) {