BC-7830 - additional changes for board load tests (#5207)

* improve establishing of connections
* improve error handling if token and/or target-url are not defined or not working
* add retries if establishing of connections fails
* change fileextension of loadtest report json files and add to .gitignore

.gitignore

@@ -7,6 +7,7 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
+*.loadtest.json
 
 # Runtime data
 pids

apps/server/src/modules/board/loadtest/board-collaboration.load.spec.ts

@@ -19,6 +19,9 @@ describe('Board Collaboration Load Test', () => {
 			};
 
 			const socketConnectionManager = new SocketConnectionManager(socketConfiguration);
+			let connectionIssues = 0;
+			// eslint-disable-next-line no-plusplus
+			socketConnectionManager.setOnErrorHandler(() => connectionIssues++);
 			const createBoardLoadTest: CreateBoardLoadTest = (...args) => new BoardLoadTest(...args);
 			const runner = new LoadtestRunner(socketConnectionManager, createBoardLoadTest);
 
@@ -30,6 +33,8 @@ describe('Board Collaboration Load Test', () => {
 					{ classDefinition: collaborativeClass, amount: collabClassesAmount },
 				],
 			});
+
+			await socketConnectionManager.destroySocketConnections();
 		} else {
 			expect('this should only be ran manually').toBeTruthy();
 		}

apps/server/src/modules/board/loadtest/board-load-test.spec.ts

@@ -1,3 +1,4 @@
+import { createMock } from '@golevelup/ts-jest';
 import { BoardLoadTest } from './board-load-test';
 import { fastEditor } from './helper/class-definitions';
 import { SocketConnectionManager } from './socket-connection-manager';
@@ -11,18 +12,21 @@ jest.mock('./helper/sleep', () => {
 
 jest.mock('./loadtest-client', () => {
 	return {
-		createColumn: jest.fn().mockResolvedValue({ id: 'some-id' }),
-		createCard: jest.fn().mockResolvedValue({ id: 'some-id' }),
-		createElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
-		createAndUpdateLinkElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
-		createAndUpdateTextElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
-		updateCardTitle: jest.fn().mockResolvedValue({ id: 'some-id' }),
-		updateColumnTitle: jest.fn().mockResolvedValue({ id: 'some-id' }),
+		createLoadtestClient: () => {
+			return {
+				createColumn: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createCard: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createAndUpdateLinkElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createAndUpdateTextElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				fetchBoard: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				updateCardTitle: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				updateColumnTitle: jest.fn().mockResolvedValue({ id: 'some-id' }),
+			};
+		},
 	};
 });
 
-jest.mock('./socket-connection-manager');
-
 const testClass: ClassDefinition = {
 	name: 'viewersClass',
 	users: [{ ...fastEditor, amount: 5 }],
@@ -41,7 +45,10 @@ afterEach(() => {
 describe('BoardLoadTest', () => {
 	const setup = () => {
 		const socketConfiguration = { baseUrl: '', path: '', token: '' };
-		const socketConnectionManager = new SocketConnectionManager(socketConfiguration);
+		const socketConnectionManager = createMock<SocketConnectionManager>();
+		socketConnectionManager.createConnections = jest
+			.fn()
+			.mockResolvedValue([new SocketConnection(socketConfiguration, console.log)]);
 		const socketConnection = new SocketConnection(socketConfiguration, console.log);
 
 		const boarLoadTest = new BoardLoadTest(socketConnectionManager, console.log);
@@ -53,7 +60,11 @@ describe('BoardLoadTest', () => {
 			it('should do nothing', async () => {
 				const { boarLoadTest } = setup();
 				const boardId = 'board-id';
-				const configuration = { name: 'my-configuration', users: [], simulateUsersTimeMs: 2000 };
+				const configuration = {
+					name: 'my-configuration',
+					users: [{ name: 'tempuserprofile', isActive: true, sleepMs: 100, amount: 1 }],
+					simulateUsersTimeMs: 2000,
+				};
 
 				const response = await boarLoadTest.runBoardTest(boardId, configuration);
 
@@ -68,11 +79,33 @@ describe('BoardLoadTest', () => {
 
 				await boarLoadTest.runBoardTest(boardId, testClass);
 
-				expect(socketConnectionManager.createConnection).toHaveBeenCalledTimes(5);
+				expect(socketConnectionManager.createConnections).toHaveBeenCalledTimes(1);
 			});
 		});
 	});
 
+	describe('simulateUsersActions', () => {
+		it('should simulate actions for all users', async () => {
+			const { boarLoadTest } = setup();
+			const loadtestClient = {
+				createColumn: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createCard: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createAndUpdateLinkElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				createAndUpdateTextElement: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				updateCardTitle: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				updateColumnTitle: jest.fn().mockResolvedValue({ id: 'some-id' }),
+				fetchBoard: jest.fn().mockResolvedValue({ id: 'some-id' }),
+			} as unknown as LoadtestClient;
+			const userProfile = fastEditor;
+
+			await boarLoadTest.simulateUsersActions([loadtestClient], [userProfile]);
+
+			expect(loadtestClient.createColumn).toHaveBeenCalled();
+			expect(loadtestClient.createCard).toHaveBeenCalled();
+		});
+	});
+
 	describe('simulateUserActions', () => {
 		it('should create columns and cards', async () => {
 			const { boarLoadTest } = setup();

apps/server/src/modules/board/loadtest/board-load-test.ts

@@ -2,11 +2,12 @@
 import { duplicateUserProfiles } from './helper/class-definitions';
 import { getRandomCardTitle, getRandomLink, getRandomRichContentBody } from './helper/randomData';
 import { sleep } from './helper/sleep';
-import { LoadtestClient } from './loadtest-client';
+import { createLoadtestClient, LoadtestClient } from './loadtest-client';
+import { SocketConnection } from './socket-connection';
 import { SocketConnectionManager } from './socket-connection-manager';
 import { Callback, ClassDefinition, UserProfile } from './types';
 
-const SIMULATE_USER_TIME_MS = 60000;
+const SIMULATE_USER_TIME_MS = 120000;
 
 export class BoardLoadTest {
 	private columns: { id: string; cards: { id: string }[] }[] = [];
@@ -24,19 +25,19 @@ export class BoardLoadTest {
 	}
 
 	async initializeLoadtestClients(amount: number, boardId: string): Promise<LoadtestClient[]> {
-		const promises = Array(amount)
-			.fill(1)
-			.map(() => this.initializeLoadtestClient(boardId));
+		const connections = await this.socketConnectionManager.createConnections(amount);
+		const promises = connections.map((socketConnection: SocketConnection) =>
+			this.initializeLoadtestClient(socketConnection, boardId)
+		);
 		const results = await Promise.all(promises);
 		return results;
 	}
 
-	async initializeLoadtestClient(boardId: string): Promise<LoadtestClient> {
-		const socketConnection = await this.socketConnectionManager.createConnection();
-		const loadtestClient = new LoadtestClient(socketConnection, boardId);
-
+	async initializeLoadtestClient(socketConnection: SocketConnection, boardId: string): Promise<LoadtestClient> {
 		/* istanbul ignore next */
-		await sleep(Math.ceil(Math.random() * 3000));
+		const loadtestClient = createLoadtestClient(socketConnection, boardId);
+
+		await sleep(Math.ceil(Math.random() * 20000));
 		/* istanbul ignore next */
 		await loadtestClient.fetchBoard();
 		/* istanbul ignore next */

apps/server/src/modules/board/loadtest/connection.load.spec.ts

@@ -24,7 +24,7 @@ describe('Board Collaboration - Connection Load Test', () => {
 			const sockets = await manager.createConnections(CONNECTION_AMOUNT);
 			await sleep(3000);
 			expect(sockets).toHaveLength(CONNECTION_AMOUNT);
-			await manager.destroySocketConnections(sockets);
+			await manager.destroySocketConnections();
 		} else {
 			expect('this should only be ran manually').toBeTruthy();
 		}

apps/server/src/modules/board/loadtest/helper/create-board.spec.ts

@@ -1,18 +1,19 @@
-import { createBoard } from './create-board';
+import { createBoard, createBoardsResilient } from './create-board';
 
 describe('createBoards', () => {
+	const mockFetch = jest.fn();
+
+	beforeEach(() => {
+		global.fetch = mockFetch;
+	});
+
+	afterEach(() => {
+		jest.clearAllMocks();
+	});
+
 	describe('createBoard', () => {
 		const apiBaseUrl = 'http://example.com';
 		const courseId = 'course123';
-		const mockFetch = jest.fn();
-
-		beforeEach(() => {
-			global.fetch = mockFetch;
-		});
-
-		afterEach(() => {
-			jest.clearAllMocks();
-		});
 
 		it('should create a board and return its id', async () => {
 			const boardId = 'board123';
@@ -48,9 +49,54 @@ describe('createBoards', () => {
 				},
 			});
 
-			await expect(createBoard(apiBaseUrl, token, courseId)).rejects.toThrow(
-				'Failed to create board: 400 - check token and target in env-variables'
-			);
+			await expect(createBoard(apiBaseUrl, token, courseId)).rejects.toThrow();
+		});
+	});
+
+	describe('createBoardsResilient', () => {
+		it('should create the correct amount of boards', async () => {
+			mockFetch.mockResolvedValue({
+				status: 201,
+				json: () => {
+					return { id: `board${Math.ceil(Math.random() * 1000)}` };
+				},
+			});
+
+			await createBoardsResilient('http://example.com', 'test-token', 'course123', 5);
+
+			expect(mockFetch).toHaveBeenCalledTimes(5);
+		});
+
+		it('should retry on error and in the end return the possible amount of boardIds', async () => {
+			mockFetch.mockResolvedValueOnce({
+				status: 201,
+				json: () => {
+					return { id: `board${Math.ceil(Math.random() * 1000)}` };
+				},
+			});
+
+			mockFetch.mockResolvedValue({
+				status: 404,
+				json: () => {
+					return {};
+				},
+			});
+
+			const boardIds = await createBoardsResilient('http://example.com', 'test-token', 'course123', 5);
+
+			expect(mockFetch).toHaveBeenCalledTimes(11);
+			expect(boardIds).toHaveLength(1);
+		});
+
+		it('should throw an error if the token is unauthorized', async () => {
+			mockFetch.mockResolvedValue({
+				status: 401,
+				json: () => {
+					return {};
+				},
+			});
+
+			await expect(createBoardsResilient('http://example.com', 'test-token', 'course123', 5)).rejects.toThrow();
 		});
 	});
 });

apps/server/src/modules/board/loadtest/helper/create-board.ts

@@ -1,4 +1,11 @@
 import { BoardExternalReferenceType, BoardLayout } from '../../domain';
+import { sleep } from './sleep';
+
+export class HttpError extends Error {
+	constructor(public readonly status: number, public readonly statusText: string) {
+		super(`HTTP Error ${status}: ${statusText}`);
+	}
+}
 
 export const createBoard = async (apiBaseUrl: string, token: string, courseId: string) => {
 	const boardTitle = `${new Date().toISOString().substring(0, 10)} ${new Date().toLocaleTimeString(
@@ -22,8 +29,31 @@ export const createBoard = async (apiBaseUrl: string, token: string, courseId: s
 	});
 
 	if (response.status !== 201) {
-		throw new Error(`Failed to create board: ${response.status} - check token and target in env-variables`);
+		throw new HttpError(response.status, response.statusText);
 	}
 	const body = (await response.json()) as unknown as { id: string };
 	return body.id;
 };
+
+export const createBoardsResilient = async (apiBaseUrl: string, token: string, courseId: string, amount: number) => {
+	const boardIds: string[] = [];
+	let retries = 0;
+	while (boardIds.length < amount && retries < 10) {
+		try {
+			// eslint-disable-next-line no-await-in-loop
+			const boardId = await createBoard(apiBaseUrl, token, courseId);
+			boardIds.push(boardId);
+		} catch (err) {
+			if ('status' in err) {
+				const { status } = err as unknown as HttpError;
+				if (status === 401) {
+					throw new Error('Unauthorized REST-Api access - check token, url and courseId in environment variables.');
+				}
+			}
+			retries += 1;
+			// eslint-disable-next-line no-await-in-loop
+			await sleep(100);
+		}
+	}
+	return boardIds;
+};

apps/server/src/modules/board/loadtest/loadtest-client.ts

@@ -136,3 +136,5 @@ export class LoadtestClient {
 		return result as UpdateContentElementMessageParams;
 	}
 }
+
+export const createLoadtestClient = (socket: SocketConnection, boardId: string) => new LoadtestClient(socket, boardId);

apps/server/src/modules/board/loadtest/loadtest-runner.spec.ts

@@ -7,7 +7,7 @@ jest.mock('./socket-connection-manager');
 
 jest.mock('./helper/create-board', () => {
 	return {
-		createBoard: jest.fn().mockResolvedValue({ id: 'board123' }),
+		createBoardsResilient: jest.fn().mockResolvedValue([{ id: 'board123' }]),
 	};
 });
 

apps/server/src/modules/board/loadtest/loadtest-runner.ts

@@ -2,7 +2,7 @@
 import { writeFileSync } from 'fs';
 import { Injectable } from '@nestjs/common';
 import { createSeveralClasses } from './helper/class-definitions';
-import { createBoard } from './helper/create-board';
+import { createBoardsResilient } from './helper/create-board';
 import { formatDate } from './helper/format-date';
 import { getUrlConfiguration } from './helper/get-url-configuration';
 import { useResponseTimes } from './helper/responseTimes.composable';
@@ -56,7 +56,7 @@ export class LoadtestRunner {
 	}
 
 	startRegularStats = () => {
-		this.intervalHandle = setInterval(() => this.showStats(), 10000);
+		this.intervalHandle = setInterval(() => this.showStats(), 2000);
 	};
 
 	stopRegularStats = () => {
@@ -75,7 +75,7 @@ export class LoadtestRunner {
 		socketConfiguration: SocketConfiguration,
 		configurations: ClassDefinitionWithAmount[]
 	) {
-		const protocolFilename = `${formatDate(this.startDate)}_${Math.ceil(Math.random() * 1000)}.json`;
+		const protocolFilename = `${formatDate(this.startDate)}_${Math.ceil(Math.random() * 1000)}.loadtest.json`;
 		const protocol = {
 			protocolFilename,
 			startDateTime: formatDate(this.startDate),
@@ -111,9 +111,23 @@ export class LoadtestRunner {
 
 		this.startRegularStats();
 
-		const promises: Promise<unknown>[] = classes.flatMap(async (classDefinition) => {
+		const boardIds = await createBoardsResilient(urls.api, socketConfiguration.token, courseId, classes.length).catch(
+			(err) => {
+				/* istanbul ignore next */
+				this.stopRegularStats();
+				/* istanbul ignore next */
+				throw err;
+			}
+		);
+
+		if (boardIds.length !== classes.length) {
+			/* istanbul ignore next */
+			throw new Error('Failed to create all boards');
+		}
+
+		const promises: Promise<unknown>[] = classes.flatMap(async (classDefinition, index) => {
 			const boardLoadTest = this.createBoardLoadTest(this.socketConnectionManager, this.onError);
-			const boardId = await createBoard(urls.api, socketConfiguration.token, courseId);
+			const boardId = boardIds[index];
 			return boardLoadTest.runBoardTest(boardId, classDefinition);
 		});
 

apps/server/src/modules/board/loadtest/socket-connection-manager.spec.ts

@@ -22,9 +22,11 @@ describe('SocketConnectionManager', () => {
 	describe('destroySocketConnections', () => {
 		it('should destroy the connections', async () => {
 			const { socketConnectionManager } = setup();
-			const connections = await socketConnectionManager.createConnections(5);
+			await socketConnectionManager.createConnections(5);
+
+			expect(socketConnectionManager.getClientCount()).toBe(5);
 
-			await socketConnectionManager.destroySocketConnections(connections);
+			await socketConnectionManager.destroySocketConnections();
 
 			expect(socketConnectionManager.getClientCount()).toBe(0);
 		});