diff --git a/backend/config/default.cjs b/backend/config/default.cjs index 483875f9d..69f3d9a4b 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -137,6 +137,15 @@ module.exports = { registry: { host: 'localhost:8080', }, + + inference: { + enabled: true, + connection: { + host: 'example.com', + }, + + gpus: {}, + }, }, connectors: { diff --git a/backend/src/connectors/audit/Base.ts b/backend/src/connectors/audit/Base.ts index 0d7b05c51..a64987d83 100644 --- a/backend/src/connectors/audit/Base.ts +++ b/backend/src/connectors/audit/Base.ts @@ -2,6 +2,7 @@ import { Request } from 'express' import { AccessRequestDoc } from '../../models/AccessRequest.js' import { FileInterface, FileInterfaceDoc } from '../../models/File.js' +import { InferenceDoc } from '../../models/Inference.js' import { ModelCardInterface, ModelDoc, ModelInterface } from '../../models/Model.js' import { ReleaseDoc } from '../../models/Release.js' import { ReviewInterface } from '../../models/Review.js' @@ -88,6 +89,11 @@ export const AuditInfo = { UpdateSchema: { typeId: 'UpdateSchema', description: 'Schema Updated', auditKind: AuditKind.Update }, ViewModelImages: { typeId: 'ViewModelImages', description: 'Model Images Viewed', auditKind: AuditKind.View }, + + CreateInference: { typeId: 'CreateInference', description: 'Inference Service Created', auditKind: AuditKind.Create }, + UpdateInference: { typeId: 'UpdateInference', description: 'Inference Service Updated', auditKind: AuditKind.Update }, + ViewInference: { typeId: 'ViewInference', description: 'Inference Service Viewed', auditKind: AuditKind.View }, + ViewInferences: { typeId: 'ViewInferences', description: 'Inferences Viewed', auditKind: AuditKind.View }, } as const export type AuditInfoKeys = (typeof AuditInfo)[keyof typeof AuditInfo] @@ -131,6 +137,11 @@ export abstract class BaseAuditConnector { abstract onDeleteSchema(req: Request, schemaId: string) abstract onUpdateSchema(req: Request, schema: SchemaDoc) + abstract onCreateInference(req: Request, inference: InferenceDoc) + abstract onUpdateInference(req: Request, inference: InferenceDoc) + abstract onViewInference(req: Request, inference: InferenceDoc) + abstract onViewInferences(req: Request, inference: InferenceDoc[]) + abstract onViewModelImages( req: Request, modelId: string, diff --git a/backend/src/connectors/audit/__mocks__/index.ts b/backend/src/connectors/audit/__mocks__/index.ts index 562145ead..f5df75d44 100644 --- a/backend/src/connectors/audit/__mocks__/index.ts +++ b/backend/src/connectors/audit/__mocks__/index.ts @@ -36,6 +36,11 @@ const audit = { onDeleteSchema: vi.fn(), onSearchSchemas: vi.fn(), + onCreateInference: vi.fn(), + onViewInference: vi.fn(), + onUpdateInference: vi.fn(), + onViewInferences: vi.fn(), + onViewModelImages: vi.fn(), onError: vi.fn(), diff --git a/backend/src/connectors/audit/silly.ts b/backend/src/connectors/audit/silly.ts index 1c3a71be1..e4783a51a 100644 --- a/backend/src/connectors/audit/silly.ts +++ b/backend/src/connectors/audit/silly.ts @@ -3,6 +3,7 @@ import { Request } from 'express' import { AccessRequestDoc } from '../../models/AccessRequest.js' import { FileInterface, FileInterfaceDoc } from '../../models/File.js' +import { InferenceDoc } from '../../models/Inference.js' import { ModelCardInterface, ModelDoc, ModelInterface } from '../../models/Model.js' import { ReleaseDoc } from '../../models/Release.js' import { ReviewInterface } from '../../models/Review.js' @@ -49,5 +50,9 @@ export class SillyAuditConnector extends BaseAuditConnector { onUpdateSchema(_req: Request, _schema: SchemaDoc) {} onViewSchema(_req: Request, _schema: SchemaInterface) {} onViewModelImages(_req: Request, _modelId: string, _images: { repository: string; name: string; tags: string[] }[]) {} + onViewInferences(_req: Request, _inferences: InferenceDoc[]) {} + onViewInference(_req: Request, _inferences: InferenceDoc) {} + onUpdateInference(_req: Request, _inferences: InferenceDoc) {} + onCreateInference(_req: Request, _inferences: InferenceDoc) {} onError(_req: Request, _error: BailoError) {} } diff --git a/backend/src/connectors/audit/stdout.ts b/backend/src/connectors/audit/stdout.ts index 91533c7bb..2b46d82fb 100644 --- a/backend/src/connectors/audit/stdout.ts +++ b/backend/src/connectors/audit/stdout.ts @@ -2,6 +2,7 @@ import { Request } from 'express' import { AccessRequestDoc } from '../../models/AccessRequest.js' import { FileInterface, FileInterfaceDoc } from '../../models/File.js' +import { InferenceDoc } from '../../models/Inference.js' import { ModelCardInterface, ModelDoc, ModelInterface } from '../../models/Model.js' import { ReleaseDoc } from '../../models/Release.js' import { ReviewInterface } from '../../models/Review.js' @@ -274,4 +275,38 @@ export class StdoutAuditConnector extends BaseAuditConnector { }) req.log.info(event, req.audit.description) } + + onViewInference(req: Request, inference: InferenceDoc) { + this.checkEventType(AuditInfo.ViewInference, req) + const event = this.generateEvent(req, { + modelId: inference.modelId, + imageName: inference.image, + imageTag: inference.tag, + }) + req.log.info(event, req.audit.description) + } + + onViewInferences(req: Request, inferences: InferenceDoc[]) { + this.checkEventType(AuditInfo.ViewInferences, req) + const event = this.generateEvent(req, { + results: inferences.map((inference) => ({ + modelId: inference.modelId, + image: inference.image, + tag: inference.tag, + })), + }) + req.log.info(event, req.audit.description) + } + + onCreateInference(req: Request, inference: InferenceDoc) { + this.checkEventType(AuditInfo.CreateInference, req) + const event = this.generateEvent(req, { modelId: inference.modelId, image: inference.image, tag: inference.tag }) + req.log.info(event, req.audit.description) + } + + onUpdateInference(req: Request, inference: InferenceDoc) { + this.checkEventType(AuditInfo.UpdateInference, req) + const event = this.generateEvent(req, { modelId: inference.modelId, image: inference.image, tag: inference.tag }) + req.log.info(event, req.audit.description) + } } diff --git a/backend/src/models/Inference.ts b/backend/src/models/Inference.ts new file mode 100644 index 000000000..1139087e3 --- /dev/null +++ b/backend/src/models/Inference.ts @@ -0,0 +1,62 @@ +import { Document, model, Schema } from 'mongoose' + +export interface InferenceSetting { + processorType: string + memory?: number + port: number +} + +export interface InferenceInterface { + modelId: string + image: string + tag: string + + description: string + + settings: InferenceSetting + + createdBy: string + createdAt: Date + updatedAt: Date +} + +export type InferenceDoc = InferenceInterface & Document + +const InferenceSchema = new Schema( + { + modelId: { type: String, required: true }, + image: { type: String, required: true }, + tag: { type: String, required: true }, + + description: { type: String, required: false, default: '' }, + + settings: { + processorType: { type: String, required: true }, + memory: { + type: Number, + required: function (this: InferenceInterface): boolean { + return this.settings.processorType === 'cpu' + }, + validate: function (this: InferenceInterface, val: any): boolean { + if (this.settings.processorType === 'cpu' && val) { + return true + } + throw new Error(`Cannot specify memory allocation without choosing cpu as the processor type`) + }, + }, + port: { type: Number, required: true }, + }, + + createdBy: { type: String, required: true }, + }, + { + timestamps: true, + collection: 'v2_model_inferences', + }, +) + +InferenceSchema.index({ modelId: 1, image: 1, tag: 1 }, { unique: true }) + +const InferenceModel = model('v2_Model_Inference', InferenceSchema) + +export default InferenceModel diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 5c38aa780..993cfa7bc 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -24,6 +24,10 @@ import { postStartMultipartUpload } from './routes/v2/model/file/postStartMultip import { getModel } from './routes/v2/model/getModel.js' import { getModelsSearch } from './routes/v2/model/getModelsSearch.js' import { getImages } from './routes/v2/model/images/getImages.js' +import { getInference } from './routes/v2/model/inferencing/getInferenceService.js' +import { getInferences } from './routes/v2/model/inferencing/getInferenceServices.js' +import { postInference } from './routes/v2/model/inferencing/postInferenceService.js' +import { putInference } from './routes/v2/model/inferencing/putInferenceService.js' import { getModelCard } from './routes/v2/model/modelcard/getModelCard.js' import { getModelCardRevisions } from './routes/v2/model/modelcard/getModelCardRevisions.js' import { postFromSchema } from './routes/v2/model/modelcard/postFromSchema.js' @@ -140,6 +144,11 @@ server.post('/api/v2/model/:modelId/files/upload/multipart/start', ...postStartM server.post('/api/v2/model/:modelId/files/upload/multipart/finish', ...postFinishMultipartUpload) server.delete('/api/v2/model/:modelId/file/:fileId', ...deleteFile) +server.get('/api/v2/model/:modelId/inferences', ...getInferences) +server.get('/api/v2/model/:modelId/inference/:image/:tag', ...getInference) +server.post('/api/v2/model/:modelId/inference', ...postInference) +server.put('/api/v2/model/:modelId/inference/:image/:tag', ...putInference) + // *server.get('/api/v2/model/:modelId/release/:semver/file/:fileCode/list', ...getModelFileList) // *server.get('/api/v2/model/:modelId/release/:semver/file/:fileCode/raw', ...getModelFileRaw) diff --git a/backend/src/routes/v2/model/inferencing/getInferenceService.ts b/backend/src/routes/v2/model/inferencing/getInferenceService.ts new file mode 100644 index 000000000..a91bfb3e4 --- /dev/null +++ b/backend/src/routes/v2/model/inferencing/getInferenceService.ts @@ -0,0 +1,58 @@ +import bodyParser from 'body-parser' +import { Request, Response } from 'express' +import { z } from 'zod' + +import { AuditInfo } from '../../../../connectors/audit/Base.js' +import audit from '../../../../connectors/audit/index.js' +import { InferenceInterface } from '../../../../models/Inference.js' +import { getInferenceByImage } from '../../../../services/inference.js' +import { inferenceInterfaceSchema, registerPath } from '../../../../services/specification.js' +import { parse } from '../../../../utils/validate.js' + +export const getInferenceSchema = z.object({ + params: z.object({ + modelId: z.string(), + image: z.string(), + tag: z.string(), + }), +}) + +registerPath({ + method: 'get', + path: '/api/v2/model/{modelId}/inference/{image}/{tag}', + tags: ['inference'], + description: 'Get details for an inferencing service within the cluster.', + schema: getInferenceSchema, + responses: { + 200: { + description: 'Details for a specific inferencing instance.', + content: { + 'application/json': { + schema: z.object({ inference: inferenceInterfaceSchema }), + }, + }, + }, + }, +}) + +interface GetInferenceService { + inference: InferenceInterface +} + +export const getInference = [ + bodyParser.json(), + async (req: Request, res: Response) => { + req.audit = AuditInfo.ViewInference + const { + params: { modelId, image, tag }, + } = parse(req, getInferenceSchema) + + const inference = await getInferenceByImage(req.user, modelId, image, tag) + + await audit.onViewInference(req, inference) + + return res.json({ + inference, + }) + }, +] diff --git a/backend/src/routes/v2/model/inferencing/getInferenceServices.ts b/backend/src/routes/v2/model/inferencing/getInferenceServices.ts new file mode 100644 index 000000000..84da9a15d --- /dev/null +++ b/backend/src/routes/v2/model/inferencing/getInferenceServices.ts @@ -0,0 +1,56 @@ +import bodyParser from 'body-parser' +import { Request, Response } from 'express' +import { z } from 'zod' + +import { AuditInfo } from '../../../../connectors/audit/Base.js' +import audit from '../../../../connectors/audit/index.js' +import { InferenceInterface } from '../../../../models/Inference.js' +import { getInferencesByModel } from '../../../../services/inference.js' +import { inferenceInterfaceSchema, registerPath } from '../../../../services/specification.js' +import { parse } from '../../../../utils/validate.js' + +export const getInferencesSchema = z.object({ + params: z.object({ + modelId: z.string(), + }), +}) + +registerPath({ + method: 'get', + path: '/api/v2/model/{modelId}/inferences', + tags: ['inference'], + description: 'Get all of the inferencing services associated with a model.', + schema: getInferencesSchema, + responses: { + 200: { + description: 'An array of inferencing services.', + content: { + 'application/json': { + schema: z.object({ + inferences: z.array(inferenceInterfaceSchema), + }), + }, + }, + }, + }, +}) + +interface GetInferenceService { + inferences: Array +} + +export const getInferences = [ + bodyParser.json(), + async (req: Request, res: Response) => { + req.audit = AuditInfo.ViewInferences + const { params } = parse(req, getInferencesSchema) + + const inferences = await getInferencesByModel(req.user, params.modelId) + + await audit.onViewInferences(req, inferences) + + return res.json({ + inferences, + }) + }, +] diff --git a/backend/src/routes/v2/model/inferencing/postInferenceService.ts b/backend/src/routes/v2/model/inferencing/postInferenceService.ts new file mode 100644 index 000000000..45dec208d --- /dev/null +++ b/backend/src/routes/v2/model/inferencing/postInferenceService.ts @@ -0,0 +1,67 @@ +import bodyParser from 'body-parser' +import { Request, Response } from 'express' +import { z } from 'zod' + +import { AuditInfo } from '../../../../connectors/audit/Base.js' +import audit from '../../../../connectors/audit/index.js' +import { InferenceInterface } from '../../../../models/Inference.js' +import { createInference } from '../../../../services/inference.js' +import { inferenceInterfaceSchema, registerPath } from '../../../../services/specification.js' +import { parse } from '../../../../utils/validate.js' + +export const postInferenceSchema = z.object({ + params: z.object({ + modelId: z.string(), + }), + body: z.object({ + image: z.string(), + tag: z.string(), + description: z.string(), + settings: z.object({ + processorType: z.string(), + memory: z.number().optional(), + port: z.number(), + }), + }), +}) + +registerPath({ + method: 'post', + path: '/api/v2/model/{modelId}/inference', + tags: ['inference'], + description: 'Create a inferencing service within Bailo', + schema: postInferenceSchema, + responses: { + 200: { + description: 'The created inferencing service.', + content: { + 'application/json': { + schema: inferenceInterfaceSchema, + }, + }, + }, + }, +}) + +interface PostInferenceService { + inference: InferenceInterface +} + +export const postInference = [ + bodyParser.json(), + async (req: Request, res: Response) => { + req.audit = AuditInfo.CreateInference + const { + params: { modelId }, + body, + } = parse(req, postInferenceSchema) + + const inference = await createInference(req.user, modelId, body) + + await audit.onCreateInference(req, inference) + + return res.json({ + inference, + }) + }, +] diff --git a/backend/src/routes/v2/model/inferencing/putInferenceService.ts b/backend/src/routes/v2/model/inferencing/putInferenceService.ts new file mode 100644 index 000000000..2c1ebb51d --- /dev/null +++ b/backend/src/routes/v2/model/inferencing/putInferenceService.ts @@ -0,0 +1,67 @@ +import bodyParser from 'body-parser' +import { Request, Response } from 'express' +import { z } from 'zod' + +import { AuditInfo } from '../../../../connectors/audit/Base.js' +import audit from '../../../../connectors/audit/index.js' +import { InferenceInterface } from '../../../../models/Inference.js' +import { updateInference } from '../../../../services/inference.js' +import { inferenceInterfaceSchema, registerPath } from '../../../../services/specification.js' +import { parse } from '../../../../utils/validate.js' + +export const putInferenceSchema = z.object({ + params: z.object({ + modelId: z.string(), + image: z.string(), + tag: z.string(), + }), + body: z.object({ + description: z.string(), + settings: z.object({ + processorType: z.string(), + memory: z.number().optional(), + port: z.number(), + }), + }), +}) + +registerPath({ + method: 'put', + path: '/api/v2/model/{modelId}/inference/{image}/{tag}', + tags: ['inference'], + description: 'Update a inferencing service within Bailo', + schema: putInferenceSchema, + responses: { + 200: { + description: 'The created inferencing service.', + content: { + 'application/json': { + schema: inferenceInterfaceSchema, + }, + }, + }, + }, +}) + +interface PutInferenceService { + inference: InferenceInterface +} + +export const putInference = [ + bodyParser.json(), + async (req: Request, res: Response) => { + req.audit = AuditInfo.UpdateInference + const { + params: { modelId, image, tag }, + body, + } = parse(req, putInferenceSchema) + + const inference = await updateInference(req.user, modelId, image, tag, body) + + await audit.onUpdateInference(req, inference) + + return res.json({ + inference, + }) + }, +] diff --git a/backend/src/routes/v2/specification.ts b/backend/src/routes/v2/specification.ts index 1cc5c6037..e8786a17c 100644 --- a/backend/src/routes/v2/specification.ts +++ b/backend/src/routes/v2/specification.ts @@ -64,6 +64,11 @@ export const getSpecification = [ name: 'user', description: 'A user represents an individual who has accessed this service.', }, + { + name: 'inference', + description: + 'An inference service is used to run models within Bailo. Each contains settings for a specific configuration', + }, ], }), ) diff --git a/backend/src/services/inference.ts b/backend/src/services/inference.ts new file mode 100644 index 000000000..2a2933371 --- /dev/null +++ b/backend/src/services/inference.ts @@ -0,0 +1,117 @@ +import { ModelAction } from '../connectors/authorisation/actions.js' +import authorisation from '../connectors/authorisation/index.js' +import InferenceModel, { InferenceDoc, InferenceInterface } from '../models/Inference.js' +import Inference from '../models/Inference.js' +import { UserInterface } from '../models/User.js' +import { BadReq, Forbidden, NotFound } from '../utils/error.js' +import { isMongoServerError } from '../utils/mongo.js' +import { getModelById } from './model.js' +import { listModelImages } from './registry.js' + +export async function getInferenceByImage(user: UserInterface, modelId: string, image: string, tag: string) { + const model = await getModelById(user, modelId) + const auth = await authorisation.model(user, model, ModelAction.View) + if (!auth.success) { + throw Forbidden(auth.info, { userDn: user.dn, modelId }) + } + + const inference = await Inference.findOne({ + modelId, + image, + tag, + }) + + if (!inference) { + throw NotFound(`The requested inferencing service was not found.`, { modelId, image, tag }) + } + + return inference +} + +export async function getInferencesByModel(user: UserInterface, modelId: string) { + const model = await getModelById(user, modelId) + const auth = await authorisation.model(user, model, ModelAction.View) + if (!auth.success) { + throw Forbidden(auth.info, { userDn: user.dn, modelId }) + } + + const inferences = await InferenceModel.find({ modelId }) + + return inferences +} + +export type CreateInferenceParams = Pick + +export async function createInference(user: UserInterface, modelId: string, inferenceParams: CreateInferenceParams) { + const model = await getModelById(user, modelId) + + const auth = await authorisation.model(user, model, ModelAction.Create) + if (!auth.success) { + throw Forbidden(auth.info, { + userDn: user.dn, + modelId: modelId, + image: inferenceParams.image, + tag: inferenceParams.tag, + }) + } + + // Check that an image exists in the registry + const images = await listModelImages(user, modelId) + + const image = images.find((image) => image.repository === modelId && image.name === inferenceParams.image) + + if (!image?.tags.includes(inferenceParams.tag)) { + throw NotFound(`Image ${inferenceParams.image}:${inferenceParams.tag} was not found on this model.`, { + modelId: modelId, + }) + } + const inference = new Inference({ + modelId: modelId, + createdBy: user.dn, + ...inferenceParams, + }) + + try { + await inference.save() + } catch (error) { + if (isMongoServerError(error) && error.code === 11000) { + throw BadReq(`A service with this image already exists.`, { + modelId: modelId, + image: inferenceParams.image, + tag: inferenceParams.tag, + }) + } + + throw error + } + + return inference +} + +export type UpdateInferenceParams = Pick +export async function updateInference( + user: UserInterface, + modelId: string, + image: string, + tag: string, + inferenceParams: UpdateInferenceParams, +): Promise { + const model = await getModelById(user, modelId) + + const auth = await authorisation.model(user, model, ModelAction.Update) + + if (!auth.success) { + throw Forbidden(auth.info, { userDn: user.dn, modelId: modelId }) + } + const inference = await getInferenceByImage(user, modelId, image, tag) + + const updatedInference = await Inference.findOneAndUpdate( + { modelId: inference.modelId, image: inference.image, tag: inference.tag }, + inferenceParams, + { new: true }, + ) + if (!updatedInference) { + throw NotFound(`The requested inference service was not found.`, { updatedInference }) + } + return updatedInference +} diff --git a/backend/src/services/specification.ts b/backend/src/services/specification.ts index e0ea69566..a6110d677 100644 --- a/backend/src/services/specification.ts +++ b/backend/src/services/specification.ts @@ -204,6 +204,23 @@ export const schemaInterfaceSchema = z.object({ updatedAt: z.string().openapi({ example: new Date().toISOString() }), }) +export const inferenceInterfaceSchema = z.object({ + modelId: z.string().openapi({ example: 'yolo-v4-abcdef' }), + image: z.string().openapi({ example: 'yolov4' }), + tag: z.string().openapi({ example: 'latest' }), + + description: z.string().openapi({ example: 'A deployment for running Yolo V4 in Bailo' }), + + settings: z.object({ + processorType: z.string().openapi({ example: 'cpu' }), + memory: z.number().optional().openapi({ example: 4 }), + port: z.number().openapi({ example: 8080 }), + }), + + createdAt: z.string().openapi({ example: new Date().toISOString() }), + updatedAt: z.string().openapi({ example: new Date().toISOString() }), +}) + export const userTokenSchema = z.object({ description: z.string().openapi({ example: 'user token' }), diff --git a/backend/src/utils/config.ts b/backend/src/utils/config.ts index bc66ce116..904b2aeac 100644 --- a/backend/src/utils/config.ts +++ b/backend/src/utils/config.ts @@ -113,6 +113,15 @@ export interface Config { registry: { host: string } + + inference: { + enabled: boolean + connection: { + host: string + } + + gpus: { [key: string]: string } + } } session: { diff --git a/backend/test/routes/model/inferencing/__snapshots__/createInferencingService.spec.ts.snap b/backend/test/routes/model/inferencing/__snapshots__/createInferencingService.spec.ts.snap new file mode 100644 index 000000000..a9412e6f3 --- /dev/null +++ b/backend/test/routes/model/inferencing/__snapshots__/createInferencingService.spec.ts.snap @@ -0,0 +1,16 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`routes > inferencing > postInference > 200 > ok 1`] = ` +{ + "inference": { + "_id": "test", + }, +} +`; + +exports[`routes > inferencing > postInference > audit > expected call 1`] = ` +{ + "_id": "test", + "toObject": [MockFunction spy], +} +`; diff --git a/backend/test/routes/model/inferencing/__snapshots__/getInferencingService.spec.ts.snap b/backend/test/routes/model/inferencing/__snapshots__/getInferencingService.spec.ts.snap new file mode 100644 index 000000000..cc5e262a4 --- /dev/null +++ b/backend/test/routes/model/inferencing/__snapshots__/getInferencingService.spec.ts.snap @@ -0,0 +1,16 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`routes > inferencing > getInference > 200 > ok 1`] = ` +{ + "inference": { + "_id": "test", + }, +} +`; + +exports[`routes > inferencing > getInference > audit > expected call 1`] = ` +{ + "_id": "test", + "toObject": [MockFunction spy], +} +`; diff --git a/backend/test/routes/model/inferencing/__snapshots__/getInferencingServices.spec.ts.snap b/backend/test/routes/model/inferencing/__snapshots__/getInferencingServices.spec.ts.snap new file mode 100644 index 000000000..024d63e0d --- /dev/null +++ b/backend/test/routes/model/inferencing/__snapshots__/getInferencingServices.spec.ts.snap @@ -0,0 +1,19 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`routes > inferencing > getInferences > 200 > ok 1`] = ` +{ + "inferences": [ + { + "_id": "test", + }, + ], +} +`; + +exports[`routes > inferencing > getInferences > audit > expected call 1`] = ` +[ + { + "_id": "test", + }, +] +`; diff --git a/backend/test/routes/model/inferencing/__snapshots__/updateInferencingService.spec.ts.snap b/backend/test/routes/model/inferencing/__snapshots__/updateInferencingService.spec.ts.snap new file mode 100644 index 000000000..ed693d8f5 --- /dev/null +++ b/backend/test/routes/model/inferencing/__snapshots__/updateInferencingService.spec.ts.snap @@ -0,0 +1,16 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`routes > inferencing > updateInference > 200 > ok 1`] = ` +{ + "inference": { + "_id": "test", + }, +} +`; + +exports[`routes > inferencing > updateInference > audit > expected call 1`] = ` +{ + "_id": "test", + "toObject": [MockFunction spy], +} +`; diff --git a/backend/test/routes/model/inferencing/createInferencingService.spec.ts b/backend/test/routes/model/inferencing/createInferencingService.spec.ts new file mode 100644 index 000000000..bb73b4c82 --- /dev/null +++ b/backend/test/routes/model/inferencing/createInferencingService.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, test, vi } from 'vitest' + +import audit from '../../../../src/connectors/audit/__mocks__/index.js' +import { postInferenceSchema } from '../../../../src/routes/v2/model/inferencing/postInferenceService.js' +import { createFixture, testPost } from '../../../testUtils/routes.js' + +vi.mock('../../../../src/utils/config.js') +vi.mock('../../../../src/utils/user.js') +vi.mock('../../../../src/connectors/audit/index.js') +vi.mock('../../../../src/connectors/authorisation/index.js') + +vi.mock('../../../../src/services/inference.js', () => ({ + createInference: vi.fn(() => ({ _id: 'test', toObject: vi.fn(() => ({ _id: 'test' })) })), +})) + +describe('routes > inferencing > postInference', () => { + test('200 > ok', async () => { + const fixture = createFixture(postInferenceSchema) + const res = await testPost(`/api/v2/model/${fixture.params.modelId}/inference`, fixture) + + expect(res.statusCode).toBe(200) + expect(res.body).matchSnapshot() + }) + + test('audit > expected call', async () => { + const fixture = createFixture(postInferenceSchema) + const res = await testPost(`/api/v2/model/${fixture.params.modelId}/inference`, fixture) + + expect(res.statusCode).toBe(200) + expect(audit.onCreateInference).toBeCalled() + expect(audit.onCreateInference.mock.calls.at(0).at(1)).toMatchSnapshot() + }) +}) diff --git a/backend/test/routes/model/inferencing/getInferencingService.spec.ts b/backend/test/routes/model/inferencing/getInferencingService.spec.ts new file mode 100644 index 000000000..2d6581814 --- /dev/null +++ b/backend/test/routes/model/inferencing/getInferencingService.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, test, vi } from 'vitest' + +import audit from '../../../../src/connectors/audit/__mocks__/index.js' +import { getInferenceSchema } from '../../../../src/routes/v2/model/inferencing/getInferenceService.js' +import { createFixture, testGet } from '../../../testUtils/routes.js' + +vi.mock('../../../../src/utils/config.js') +vi.mock('../../../../src/utils/user.js') +vi.mock('../../../../src/connectors/audit/index.js') +vi.mock('../../../../src/connectors/authorisation/index.js') + +vi.mock('../../../../src/services/inference.js', () => ({ + getInferenceByImage: vi.fn(() => ({ _id: 'test', toObject: vi.fn(() => ({ _id: 'test' })) })), +})) + +describe('routes > inferencing > getInference', () => { + test('200 > ok', async () => { + const fixture = createFixture(getInferenceSchema) + const res = await testGet(`/api/v2/model/${fixture.params.modelId}/inference/example/${fixture.params.tag}`) + + expect(res.statusCode).toBe(200) + expect(res.body).matchSnapshot() + }) + + test('audit > expected call', async () => { + const fixture = createFixture(getInferenceSchema) + const res = await testGet(`/api/v2/model/${fixture.params.modelId}/inference/example/${fixture.params.tag}`) + + expect(res.statusCode).toBe(200) + expect(audit.onViewInference).toBeCalled() + expect(audit.onViewInference.mock.calls.at(0).at(1)).toMatchSnapshot() + }) +}) diff --git a/backend/test/routes/model/inferencing/getInferencingServices.spec.ts b/backend/test/routes/model/inferencing/getInferencingServices.spec.ts new file mode 100644 index 000000000..b193fa489 --- /dev/null +++ b/backend/test/routes/model/inferencing/getInferencingServices.spec.ts @@ -0,0 +1,32 @@ +import { describe, expect, test, vi } from 'vitest' + +import audit from '../../../../src/connectors/audit/__mocks__/index.js' +import { getInferencesSchema } from '../../../../src/routes/v2/model/inferencing/getInferenceServices.js' +import { createFixture, testGet } from '../../../testUtils/routes.js' + +vi.mock('../../../../src/utils/config.js') +vi.mock('../../../../src/utils/user.js') +vi.mock('../../../../src/connectors/audit/index.js') +vi.mock('../../../../src/connectors/authorisation/index.js') + +vi.mock('../../../../src/services/inference.js', () => ({ + getInferencesByModel: vi.fn(() => [{ _id: 'test' }]), +})) + +describe('routes > inferencing > getInferences', () => { + test('200 > ok', async () => { + const fixture = createFixture(getInferencesSchema) + const res = await testGet(`/api/v2/model/${fixture.params.modelId}/inferences`) + expect(res.statusCode).toBe(200) + expect(res.body).matchSnapshot() + }) + + test('audit > expected call', async () => { + const fixture = createFixture(getInferencesSchema) + const res = await testGet(`/api/v2/model/${fixture.params.modelId}/inferences`) + + expect(res.statusCode).toBe(200) + expect(audit.onViewInferences).toBeCalled() + expect(audit.onViewInferences.mock.calls.at(0).at(1)).toMatchSnapshot() + }) +}) diff --git a/backend/test/routes/model/inferencing/updateInferencingService.spec.ts b/backend/test/routes/model/inferencing/updateInferencingService.spec.ts new file mode 100644 index 000000000..e531eaa81 --- /dev/null +++ b/backend/test/routes/model/inferencing/updateInferencingService.spec.ts @@ -0,0 +1,38 @@ +import { describe, expect, test, vi } from 'vitest' + +import audit from '../../../../src/connectors/audit/__mocks__/index.js' +import { putInferenceSchema } from '../../../../src/routes/v2/model/inferencing/putInferenceService.js' +import { createFixture, testPut } from '../../../testUtils/routes.js' + +vi.mock('../../../../src/utils/config.js') +vi.mock('../../../../src/utils/user.js') +vi.mock('../../../../src/connectors/audit/index.js') +vi.mock('../../../../src/connectors/authorisation/index.js') + +vi.mock('../../../../src/services/inference.js', () => ({ + updateInference: vi.fn(() => ({ _id: 'test', toObject: vi.fn(() => ({ _id: 'test' })) })), +})) + +describe('routes > inferencing > updateInference', () => { + test('200 > ok', async () => { + const fixture = createFixture(putInferenceSchema) + const res = await testPut( + `/api/v2/model/${fixture.params.modelId}/inference/example/${fixture.params.tag}`, + fixture, + ) + expect(res.statusCode).toBe(200) + expect(res.body).matchSnapshot() + }) + + test('audit > expected call', async () => { + const fixture = createFixture(putInferenceSchema) + const res = await testPut( + `/api/v2/model/${fixture.params.modelId}/inference/example/${fixture.params.tag}`, + fixture, + ) + + expect(res.statusCode).toBe(200) + expect(audit.onUpdateInference).toBeCalled() + expect(audit.onUpdateInference.mock.calls.at(0).at(1)).toMatchSnapshot() + }) +}) diff --git a/backend/test/services/__snapshots__/inference.spec.ts.snap b/backend/test/services/__snapshots__/inference.spec.ts.snap new file mode 100644 index 000000000..b3c5f6bdb --- /dev/null +++ b/backend/test/services/__snapshots__/inference.spec.ts.snap @@ -0,0 +1,108 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`services > inference > getInferenceByModel > good 1`] = ` +[ + { + "image": "nginx", + "tag": "latest", + }, + { + "image": "yolov4", + "tag": "latest", + }, +] +`; + +exports[`services > inference > getInferenceByModel > good 2`] = ` +[ + { + "image": "nginx", + "tag": "latest", + }, + { + "image": "yolov4", + "tag": "latest", + }, +] +`; + +exports[`services > inference > getInferenceByModel > no 1`] = ` +[ + { + "image": "nginx", + "tag": "latest", + }, + { + "image": "yolov4", + "tag": "latest", + }, +] +`; + +exports[`services > inference > getInferenceByModel > no model 1`] = ` +[ + { + "image": "nginx", + "tag": "latest", + }, + { + "image": "yolov4", + "tag": "latest", + }, +] +`; + +exports[`services > inference > getInferenceByModel > noModel 1`] = ` +[ + { + "image": "nginx", + "tag": "latest", + }, + { + "image": "yolov4", + "tag": "latest", + }, +] +`; + +exports[`services > release > getModelReleases > good 1`] = ` +[ + { + "modelId": "modelId", + }, +] +`; + +exports[`services > release > getModelReleases > good 2`] = ` +[ + { + "updatedAt": -1, + }, +] +`; + +exports[`services > release > getModelReleases > good 3`] = ` +[ + { + "as": "model", + "foreignField": "id", + "from": "v2_models", + "localField": "modelId", + }, +] +`; + +exports[`services > release > getModelReleases > good 4`] = ` +[ + { + "$set": { + "model": { + "$arrayElemAt": [ + "$model", + 0, + ], + }, + }, + }, +] +`; diff --git a/backend/test/services/inference.spec.ts b/backend/test/services/inference.spec.ts new file mode 100644 index 000000000..901304b47 --- /dev/null +++ b/backend/test/services/inference.spec.ts @@ -0,0 +1,174 @@ +import { MongoServerError } from 'mongodb' +import { describe, expect, test, vi } from 'vitest' + +import authorisation from '../../src/connectors/authorisation/index.js' +import { + createInference, + getInferenceByImage, + getInferencesByModel, + updateInference, +} from '../../src/services/inference.js' + +vi.mock('../../src/connectors/authorisation/index.js') + +const modelMocks = vi.hoisted(() => ({ + getModelById: vi.fn(), + getModelCardRevision: vi.fn(), +})) +vi.mock('../../src/services/model.js', () => modelMocks) + +const registryMocks = vi.hoisted(() => ({ + listModelImages: vi.fn(() => [{ repository: 'test', name: 'nginx', tags: ['latest'] }]), +})) +vi.mock('../../src/services/registry.js', () => registryMocks) + +const inference = { + image: 'nginx', + tag: 'latest', + description: 'test', + settings: { + processorType: 'cpu', + port: 8000, + memory: 1, + }, +} + +const inferenceModelMocks = vi.hoisted(() => { + const obj: any = {} + + obj.aggregate = vi.fn(() => obj) + obj.match = vi.fn(() => obj) + obj.sort = vi.fn(() => obj) + obj.lookup = vi.fn(() => obj) + obj.append = vi.fn(() => obj) + obj.find = vi.fn(() => obj) + obj.findOne = vi.fn(() => obj) + obj.updateOne = vi.fn(() => obj) + obj.updateMany = vi.fn(() => obj) + obj.save = vi.fn(() => obj) + obj.delete = vi.fn(() => obj) + obj.findOneAndUpdate = vi.fn(() => obj) + obj.filter = vi.fn(() => obj) + + const model: any = vi.fn((params) => ({ ...obj, ...params })) + Object.assign(model, obj) + + return model +}) +vi.mock('../../src/models/Inference.js', () => ({ default: inferenceModelMocks })) + +describe('services > inference', () => { + test('createInference > simple', async () => { + const mockUser: any = { dn: 'test' } + await createInference(mockUser, 'test', inference) + + expect(inferenceModelMocks.save).toBeCalled() + expect(inferenceModelMocks).toBeCalled() + }) + + test('createInference > non-existent image', async () => { + const mockUser: any = { dn: 'test' } + expect(() => + createInference(mockUser, 'test', { image: 'non-existent', tag: 'image' } as any), + ).rejects.toThrowError(/^Image non-existent:image was not found on this model./) + }) + + test('createInference > bad authorisation', async () => { + vi.mocked(authorisation.model).mockResolvedValue({ info: 'You do not have permission', success: false, id: '' }) + + expect(() => createInference({} as any, 'modelId', inference)).rejects.toThrowError(/^You do not have permission/) + }) + + test('createInference > existing service', async () => { + const mongoError = new MongoServerError({}) + mongoError.code = 11000 + mongoError.keyValue = { + mockKey: 'mockValue', + } + inferenceModelMocks.save.mockRejectedValueOnce(mongoError) + expect(() => createInference({} as any, 'test', inference)).rejects.toThrowError( + /^A service with this image already exists./, + ) + }) + + test('updateInference > success', async () => { + await updateInference({} as any, 'test', 'nginx', 'latest', { + description: 'New description', + settings: { + port: 8000, + memory: 4, + processorType: 'cpu', + }, + }) + expect(inferenceModelMocks.findOneAndUpdate).toBeCalled() + }) + + test('updateInference > inference not found', async () => { + vi.mocked(inferenceModelMocks.findOneAndUpdate).mockResolvedValueOnce() + expect(() => + updateInference({} as any, 'test', 'non-existent', 'image', { + description: 'New description', + settings: { + port: 8000, + memory: 4, + processorType: 'cpu', + }, + }), + ).rejects.toThrowError(/^The requested inference service was not found./) + }) + + test('updateInference > bad authorisation', async () => { + vi.mocked(authorisation.model).mockResolvedValue({ info: 'You do not have permission', success: false, id: '' }) + + expect(() => + updateInference({} as any, 'test', 'nginx', 'latest', { + description: 'New description', + settings: { + port: 8000, + memory: 4, + processorType: 'cpu', + }, + }), + ).rejects.toThrowError(/^You do not have permission/) + }) + + test('getInferenceByImage > good', async () => { + await getInferenceByImage({} as any, 'test', 'nginx', 'latest') + expect(inferenceModelMocks.findOne).toBeCalled() + }) + + test('getInferenceByImage > no permission', async () => { + inferenceModelMocks.findOne.mockResolvedValue({ image: 'nginx', tag: 'latest' }) + + vi.mocked(authorisation.model).mockResolvedValue({ + info: 'You do not have permission to view this inference.', + success: false, + id: '', + }) + expect(() => getInferenceByImage({} as any, 'test', 'nginx', 'latest')).rejects.toThrowError( + /^You do not have permission to view this inference./, + ) + }) + test('getInferenceByImage > no inference', async () => { + inferenceModelMocks.findOne.mockResolvedValueOnce(undefined) + expect(() => getInferenceByImage({} as any, 'test', 'nginx', 'latest')).rejects.toThrowError( + /^The requested inferencing service was not found./, + ) + }) + + test('getInferenceByModel > good', async () => { + inferenceModelMocks.find.mockResolvedValue([ + { image: 'nginx', tag: 'latest' }, + { image: 'yolov4', tag: 'latest' }, + ]) + + const inference = await getInferencesByModel({} as any, 'modelId') + expect(inference).toMatchSnapshot() + }) + + test('getInferenceByModel > bad authorisation', async () => { + vi.mocked(authorisation.model).mockResolvedValue({ info: 'You do not have permission', success: false, id: '' }) + + expect(() => getInferencesByModel({} as any, 'modelId')).rejects.toThrowError(/^You do not have permission/) + }) +}) diff --git a/backend/vitest.config.ts b/backend/vitest.config.ts index 41d8369bb..2795c0d72 100644 --- a/backend/vitest.config.ts +++ b/backend/vitest.config.ts @@ -9,7 +9,7 @@ export default defineConfig({ include: ['test/**/*.spec.ts'], coverage: { enabled: true, - include: ['**/v2/**/*.ts', '**/middleware/**/*.ts', 'src/clients/*.ts'], + include: ['**/**/*.ts', '**/middleware/**/*.ts'], }, }, })