Skip to content

Commit

Permalink
✨Preview generation as a separate node instance
Browse files Browse the repository at this point in the history
  • Loading branch information
shepilov committed Nov 6, 2024
1 parent d86d914 commit 386bfaa
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 14 deletions.
5 changes: 5 additions & 0 deletions tdrive/backend/node/config/custom-environment-variables.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
},
"webserver": {
"host": "TWAKE_DRIVE_HOST",
"port": "TWAKE_DRIVE_PORT",
"logger": {
"level": "LOG_LEVEL"
},
Expand Down Expand Up @@ -131,5 +132,9 @@
"featureDisplayEmail": "ENABLE_FEATURE_DISPLAY_EMAIL",
"featureUserQuota": "ENABLE_FEATURE_USER_QUOTA",
"featureManageAccess": "ENABLE_FEATURE_MANAGE_ACCESS"
},
"services": {
"__name": "SERVICES_LIST",
"__format": "json"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,6 @@ export class MessageQueueService implements MessageQueueServiceAPI {

@SkipCLI()
async init(): Promise<this> {
logger.info("Initializing message-queue adapter %o", this.adapter.type);
await this.adapter?.init?.();

return this;
}

Expand All @@ -74,7 +71,8 @@ export class MessageQueueService implements MessageQueueServiceAPI {
logger.info("Starting message-queue adapter %o", this.adapter.type);
await this.adapter?.start?.();
await this.processor.start();

logger.info("Initializing message-queue adapter %o", this.adapter.type);
await this.adapter?.init?.();
return this;
}

Expand Down
6 changes: 0 additions & 6 deletions tdrive/backend/node/src/services/global-resolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import { CronAPI } from "../core/platform/services/cron/api";
import { DatabaseServiceAPI } from "../core/platform/services/database/api";
import EmailPusherAPI from "../core/platform/services/email-pusher/provider";
import { MessageQueueServiceAPI } from "../core/platform/services/message-queue/api";
import { PushServiceAPI } from "../core/platform/services/push/api";
import { SearchServiceAPI } from "../core/platform/services/search/api";
import StorageAPI from "../core/platform/services/storage/provider";
import TrackerAPI from "../core/platform/services/tracker/provider";
Expand All @@ -31,15 +30,13 @@ import { UserExternalLinksServiceImpl } from "./user/services/external_links";
import { UserServiceImpl } from "./user/services/users/service";
import { WorkspaceServiceImpl } from "./workspaces/services/workspace";

import { PreviewEngine } from "./previews/services/files/engine";
import { I18nService } from "./i18n";

type PlatformServices = {
auth: AuthServiceAPI;
counter: CounterAPI;
cron: CronAPI;
messageQueue: MessageQueueServiceAPI;
push: PushServiceAPI;
search: SearchServiceAPI;
storage: StorageAPI;
tracker: TrackerAPI;
Expand Down Expand Up @@ -91,7 +88,6 @@ class GlobalResolver {
counter: platform.getProvider<CounterAPI>("counter"),
cron: platform.getProvider<CronAPI>("cron"),
messageQueue: platform.getProvider<MessageQueueServiceAPI>("message-queue"),
push: platform.getProvider<PushServiceAPI>("push"),
search: platform.getProvider<SearchServiceAPI>("search"),
storage: platform.getProvider<StorageAPI>("storage"),
tracker: platform.getProvider<TrackerAPI>("tracker"),
Expand All @@ -106,8 +102,6 @@ class GlobalResolver {
assert(service, `Platform service ${key} was not initialized`);
});

await new PreviewEngine().init();

this.services = {
workspaces: await new WorkspaceServiceImpl().init(),
companies: await new CompanyServiceImpl().init(),
Expand Down
12 changes: 11 additions & 1 deletion tdrive/backend/node/src/services/previews/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import { Prefix, TdriveService } from "../../core/platform/framework";
import { Consumes, Prefix, TdriveService } from "../../core/platform/framework";
import { PreviewEngine } from "./services/files/engine";
import { MessageQueueServiceAPI } from "src/core/platform/services/message-queue/api";

@Prefix("/internal/services/previews/v1")
@Consumes(["message-queue", "files"])
export default class PreviewsService extends TdriveService<undefined> {
version = "1";
name = "previews";
Expand All @@ -13,4 +16,11 @@ export default class PreviewsService extends TdriveService<undefined> {
api(): undefined {
return undefined;
}

async doStart(): Promise<this> {
await new PreviewEngine(
this.context.getProvider<MessageQueueServiceAPI>("message-queue").processor,
).init();
return this;
}
}
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
import { Initializable } from "../../../../../core/platform/framework";
import { ClearProcessor } from "./clear";
import { PreviewProcessor } from "./service";
import gr from "../../../../global-resolver";
import { Processor } from "src/core/platform/services/message-queue/processor";

/**
* The notification engine is in charge of processing data and delivering user notifications on the right place
*/
export class PreviewEngine implements Initializable {
constructor(readonly processor: Processor) {}

async init(): Promise<this> {
gr.platformServices.messageQueue.processor.addHandler(new PreviewProcessor());
gr.platformServices.messageQueue.processor.addHandler(new ClearProcessor());
this.processor.addHandler(new PreviewProcessor());
this.processor.addHandler(new ClearProcessor());
return this;
}
}
166 changes: 166 additions & 0 deletions tdrive/docker-compose.dev.rabbitmq.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
version: "3.4"

services:
mongo:
container_name: mongo
image: mongo
volumes:
- ./docker-data/mongo:/data/db
ports:
- 27017:27017
networks:
- tdrive_network

node:
build:
context: .
dockerfile: docker/tdrive-node/Dockerfile
target: development
container_name: tdrive-node
hostname: tdrive_node
ports:
- 4000:4000
- 9229:9229
environment:
- DEV=dev
- SEARCH_DRIVER=mongodb
- DB_DRIVER=mongodb
- PUBSUB_TYPE=amqp
- SERVICES_LIST=[ "webserver", "database", "tracker", "applications", "search", "storage", "message-queue", "user", "files", "auth", "documents", "counter", "cron", "general", "email-pusher" ]
- ./docker-data/documents/:/storage/
volumes:
- ./backend/node/profiles:/usr/src/app/profiles
- ./backend/node/src:/usr/src/app/src
- ./docker-data/documents/:/storage/
depends_on:
- mongo
links:
- mongo
networks:
- tdrive_network

node_preview:
build:
context: .
dockerfile: docker/tdrive-node/Dockerfile
target: development
container_name: tdrive-preview-node
hostname: tdrive_preview_node
ports:
- 4001:4001
- 9229:9229
environment:
- DEV=dev
- TWAKE_DRIVE_PORT="4001"
- SEARCH_DRIVER=mongodb
- DB_DRIVER=mongodb
- PUBSUB_TYPE=amqp
- SERVICES_LIST=[ "webserver", "previews", "database", "tracker", "applications", "search", "storage", "message-queue", "user", "files", "auth", "documents", "counter", "cron", "general", "email-pusher" ]
- ./docker-data/documents/:/storage/
volumes:
- ./backend/node/profiles:/usr/src/app/profiles
- ./backend/node/src:/usr/src/app/src
- ./docker-data/documents/:/storage/
depends_on:
- mongo
links:
- mongo
networks:
- tdrive_network

frontend:
build:
context: .
dockerfile: docker/tdrive-frontend/Dockerfile
container_name: tdrive_frontend
environment:
- DEV=production
- SSL_CERTS=selfsigned
- NODE_HOST=http://tdrive_node:4000
ports:
- 80:80
- 443:443
depends_on:
- node
volumes:
- ./docker-data/logs/nginx/:/var/log/nginx
- ./docker-data/letsencrypt/:/etc/letsencrypt/
- ./docker-data/drive-preview/:/tdrive-core/web/medias/
- ./docker-data/uploads/:/tdrive-core/web/upload/
- ./docker-data/ssl:/etc/nginx/ssl
networks:
- tdrive_network

onlyoffice-connector:
build:
context: .
dockerfile: docker/onlyoffice-connector/Dockerfile
environment:
- CREDENTIALS_ENDPOINT=http://tdrive_node:4000
- ONLY_OFFICE_SERVER=http://onlyoffice:8090/
- SERVER_ORIGIN=http://onlyoffice-connector:5000
- SERVER_PORT=5000
- SERVER_PREFIX=/plugins/onlyoffice
- CREDENTIALS_ID=tdrive_onlyoffice
- CREDENTIALS_SECRET=c1cc66db78e1d3bb4713c55d5ab2
ports:
- 5000:5000
depends_on:
- onlyoffice
networks:
- tdrive_network

onlyoffice-rabbitmq:
image: rabbitmq:management
hostname: onlyoffice-rabbitmq
container_name: rabbitmq
environment:
- RABBITMQ_DEFAULT_USER=guest
- RABBITMQ_DEFAULT_PASS=guest
ports:
- "5672:5672"
- "15672:15672"
volumes:
- ./.docker-conf/rabbitmq/data/:/var/lib/rabbitmq/
- ./.docker-conf/rabbitmq/log/:/var/log/rabbitmq
networks:
- tdrive_network

onlyoffice-postgresql:
image: postgres:13
hostname: onlyoffice-postgresql
environment:
- POSTGRES_DB=onlyoffice
- POSTGRES_USER=onlyoffice
- POSTGRES_PASSWORD=onlyoffice
ports:
- 5432:5432
volumes:
- ./onlyoffice_postgres_data:/var/lib/postgresql/data
networks:
- tdrive_network

onlyoffice:
image: docker.io/onlyoffice/documentserver
ports:
- 8090:80
networks:
- tdrive_network
environment:
- AMQP_URI=amqp://guest:guest@onlyoffice-rabbitmq
- DB_HOST=onlyoffice-postgresql
- DB_NAME=onlyoffice
- DB_PORT=5432
- DB_TYPE=postgres
- DB_USER=onlyoffice
- JWT_ENABLED=false
depends_on:
- onlyoffice-rabbitmq
- onlyoffice-postgresql
volumes:
- ./onlyoffice_data:/var/www/onlyoffice/Data

networks:
tdrive_network:
driver: bridge

0 comments on commit 386bfaa

Please sign in to comment.