From fc55dd128948593cd1e7d27902e439b7b2f34cb4 Mon Sep 17 00:00:00 2001 From: lm-sec Date: Fri, 21 Jun 2024 18:10:03 -0400 Subject: [PATCH 01/11] adding the website resource and finding. It is not yet possible to see it other than with a mongodb client, but it is there! --- docs/docs/concepts/findings.md | 116 ++++++++- docs/docs/concepts/resources.md | 31 +++ .../stalker_job_sdk/__init__.py | 17 ++ .../src/modules/database/datalayer.module.ts | 4 + .../reporting/correlation.utils.spec.ts | 246 +++++++++++++++++- .../database/reporting/correlation.utils.ts | 84 +++++- .../reporting/domain/domain.summary.ts | 9 + .../database/reporting/port/port.model.ts | 2 +- .../database/reporting/port/port.summary.ts | 15 ++ .../database/reporting/project.module.ts | 2 + .../database/reporting/project.service.ts | 3 + .../websites/website-filter.model.ts | 16 ++ .../websites/website-model.module.ts | 9 + .../reporting/websites/website.controller.ts | 7 + .../reporting/websites/website.dto.ts | 0 .../reporting/websites/website.e2e-spec.ts | 171 ++++++++++++ .../reporting/websites/website.model.ts | 77 ++++++ .../reporting/websites/website.module.ts | 14 + .../websites/website.service.spec.ts | 204 +++++++++++++++ .../reporting/websites/website.service.ts | 232 +++++++++++++++++ .../subscription-triggers.module.ts | 2 + .../subscription-triggers.service.ts | 4 + .../commands/JobFindings/custom.handler.ts | 13 + .../commands/JobFindings/website.command.ts | 13 + .../commands/JobFindings/website.handler.ts | 45 ++++ .../findings/commands/findings-commands.ts | 7 + .../src/modules/findings/findings.module.ts | 2 + .../modules/findings/findings.service.spec.ts | 1 - .../src/modules/findings/findings.service.ts | 202 ++++++++------ .../src/modules/job-queue/findings-queue.ts | 5 + .../modules/job-queue/kafka-findings-queue.ts | 9 + .../modules/job-queue/null-findings-queue.ts | 4 + 32 files changed, 1452 insertions(+), 114 deletions(-) create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.summary.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-model.module.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.dto.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.module.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/website.command.ts create mode 100644 packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/website.handler.ts diff --git a/docs/docs/concepts/findings.md b/docs/docs/concepts/findings.md index 0e34788bf..86139f292 100644 --- a/docs/docs/concepts/findings.md +++ b/docs/docs/concepts/findings.md @@ -14,15 +14,16 @@ To produce a finding, the job must create an object containing the necessary inf The finding object must contain the `type` field. Here is a list of available types. -| Type | Description | -| ----------------------------------------- | -------------------------------------------------- | -| [HostnameFinding](#hostnamefinding) | Creates a new domain. | -| [IpFinding](#ipfinding) | Creates a new host. | -| [IpRangeFinding](#iprangefinding) | Creates a new IP range. | -| [HostnameIpFinding](#hostnameipfinding) | Creates a new host, attaches it to a given domain. | -| [PortFinding](#portfinding) | Creates a new port, attaches it to the given host. | -| [CustomFinding](#customfinding) | Attaches custom finding data to a given entity. | -| [PortServiceFinding](#portservicefinding) | Fills the `service` field of a port. | +| Type | Description | +| ----------------------------------------- | ------------------------------------------------------------ | +| [HostnameFinding](#hostnamefinding) | Creates a new domain. | +| [IpFinding](#ipfinding) | Creates a new host. | +| [IpRangeFinding](#iprangefinding) | Creates a new IP range. | +| [HostnameIpFinding](#hostnameipfinding) | Creates a new host, attaches it to a given domain. | +| [PortFinding](#portfinding) | Creates a new port, attaches it to the given host. | +| [WebsiteFinding](#websiteFinding) | Creates a new website, with the proper host, domain and port | +| [CustomFinding](#customfinding) | Attaches custom finding data to a given entity. | +| [PortServiceFinding](#portservicefinding) | Fills the `service` field of a port. | ## HostnameFinding @@ -186,7 +187,7 @@ Using the python SDK, you can emit this finding with the following code: ```python from stalker_job_sdk import PortFinding, log_finding port = 80 -ip = "0.0.0.0" +ip = "1.2.3.4" log_finding( PortFinding( "PortFinding", @@ -200,6 +201,60 @@ log_finding( ) ``` +## WebsiteFinding + +The `WebsiteFinding` will create a website resource. Websites are made from 4 characteristics: an IP address, a domain name, a port number and a path. Only the IP address and the port are mandatory. The domain can be empty and the path will default to `/`. + +To create a website, it must reference an existing port of a project. To reference a domain as well, it must also be a domain already known to Stalker. + +It signals that an open port running an http(s) service, either `tcp` or `udp`, has been found on the host specified through the `ip` value. The `ip` must already be known to Stalker as a valid host. A port finding creates or updates a port +and attaches it to the given host. + +> Emitting a `PortServiceFinding` with a `serviceName` of `http` and `https` will result in creating a `WebsiteFinding` per domain linked to the host, and one with an empty domain. [Learn more about PortServiceFinding and websites](#portservicefinding-and-websites) + +| Field | Description | +| -------- | ------------------------------------------------------- | +| `ip` | The ip | +| `port` | The port number | +| `domain` | The domain on which the website is hosted, can be empty | +| `path` | The folder path, defaults to `/` | + +Example: + +```json +{ + "type": "WebsiteFinding", + "key": "WebsiteFinding", + "ip": "1.2.3.4", + "port": 80, + "domain": "example.com", + "path": "/" +} +``` + +Using the python SDK, you can emit this finding with the following code: + +```python +from stalker_job_sdk import WebsiteFinding, log_finding +port = 80 +ip = "1.2.3.4" +domain = "example.com" +path = "/" + +log_finding( + WebsiteFinding( + "WebsiteFinding", + ip, + port, + domain, + path, + "New website", + [], + "WebsiteFinding", + ) +) +``` + ## CustomFinding Dynamic findings allow jobs to attach custom data to core entities. @@ -343,3 +398,44 @@ log_finding( ``` Upon receiving this finding, the backend will set the service database field of the TCP port 22 for the `0.0.0.0` IP to `ssh`. + +### PortServiceFinding and websites + +When publishing a `PortServiceFinding` with the service name of `http` or `https`, the `Jobs Manager` will understand that a website is located on that port. + +The `Jobs Manager` will therefore create and publish several `WebsiteFinding`s, one for each of the host's linked domain name, and one for the IP address alone. + +These website findings will allow further investigation of the http(s) port with the different domain names, in case the port supporting multiple virutal hosts. + +For instance, imagine a host with the IP address `1.2.3.4`. This host has the linked domains `example.com` and `dev.example.com`. + +Then, with the following code publishing the results for an https port: + +```python +from stalker_job_sdk import PortFinding, log_finding, TextField + +ip = '1.2.3.4' +port = 443 +protocol = 'tcp' +service_name = 'https' + +fields = [ + TextField("serviceName", "Service name", service_name) +] + +log_finding( + PortFinding( + "PortServiceFinding", ip, port, protocol, f"Found service {service_name}", fields + ) +) +``` + +We would create the following three websites: + +| domain | host | port | path | +| --------------- | ------- | ---- | ---- | +| N/A | 1.2.3.4 | 443 | `/` | +| example.com | 1.2.3.4 | 443 | `/` | +| dev.example.com | 1.2.3.4 | 443 | `/` | + +That way, a website at `dev.example.com`, which may be different than the one at `example.com`, will be found. The same goes for the website through direct IP access. diff --git a/docs/docs/concepts/resources.md b/docs/docs/concepts/resources.md index 5a0d55353..d31f81167 100644 --- a/docs/docs/concepts/resources.md +++ b/docs/docs/concepts/resources.md @@ -58,6 +58,33 @@ A port can be created with a `PortFinding`. A port finding is a combination of a The combination of a port's number and host identifier is unique in the database. +### Websites + +A website represents a `tcp` port running an http(s) server. + +A website is the combination between a port, a host, a domain and a path. The path, when not specified, defaults to `/`. The domain can also be empty, as not all websites have domains that resolve to them. + +A website is usually created for each http(s) port for each domain linked to a host. + +Therefore, if a host runs two http(s) ports with two domains, a total of 6 websites on the `/` path are possible. Let's take the domains `dev.example.com` and `example.com`, the IP `1.2.3.4` and the ports `80` and `443` for the `/` path. + +The following 6 values are possible: + +| domain | host | port | path | +| --------------- | ------- | ---- | ---- | +| | 1.2.3.4 | 80 | / | +| example.com | 1.2.3.4 | 80 | / | +| dev.example.com | 1.2.3.4 | 80 | / | +| | 1.2.3.4 | 443 | / | +| example.com | 1.2.3.4 | 443 | / | +| dev.example.com | 1.2.3.4 | 443 | / | + +Ports are used, combined with a *host*'s IP address, to represent a network service. Every port is linked to a *host*. They can be seen in the user interface under the `Ports` page. + +A website can be created with a `WebsiteFinding`. A website finding is a combination of an IP, a port, a domain and a path. Therefore, when a website is created, it is automatically linked to the given port, host and domain. + +The combination of a websites's port identifier, domain identifier and path is unique in the database. + ## Interacting with resources ### Tagging a resource @@ -81,3 +108,7 @@ While deleted resources are removed from the database, blocked resources will st Blocked resources can be seen in the user interface by removing the default filter `-is: blocked`. Every resource will be shown that way, blocked or not. If you wish to only see the blocked resources, use the `is: blocked` filter. Blocking a resource is useful if, through automation, Stalker found a resource that does not belong in the project. Deleting it would likely result in it reappearing later and jobs being run on it. Blocking it will ensure that jobs are not automatically run on the resource by remembering its existence. + +### Merging websites + +> This feature is not yet available, but is coming soon. diff --git a/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py b/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py index 6f2e68593..69e81c8e1 100644 --- a/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py +++ b/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py @@ -74,6 +74,23 @@ def __init__( self.port = port self.protocol = protocol +class WebsiteFinding(Finding): + def __init__( + self, + key: str, + ip: str, + port: int, + domain: str, + path: str, + name: str = None, + fields: list[Field] = [], + type: str = "CustomFinding", + ): + super().__init__(key, type, name, fields) + self.ip = ip + self.port = port + self.domain = domain + self.path = path class DomainFinding(Finding): def __init__( diff --git a/packages/backend/jobs-manager/service/src/modules/database/datalayer.module.ts b/packages/backend/jobs-manager/service/src/modules/database/datalayer.module.ts index 9fca625f6..bd542f00b 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/datalayer.module.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/datalayer.module.ts @@ -9,6 +9,7 @@ import { FindingModelModule } from './reporting/findings/findings-model.module'; import { HostModelModule } from './reporting/host/host-model.module'; import { PortModelModule } from './reporting/port/port-model.module'; import { ProjectModelModule } from './reporting/project-model.module'; +import { WebsiteModelModule } from './reporting/websites/website-model.module'; import { SecretsModelModule } from './secrets/secrets-model.module'; import { CronSubscriptionModelModule } from './subscriptions/cron-subscriptions/cron-subscription-model.module'; import { TagModelModule } from './tags/tag-model.module'; @@ -27,6 +28,7 @@ import { TagModelModule } from './tags/tag-model.module'; CronSubscriptionModelModule, SecretsModelModule, CustomJobTemplateModelModule, + WebsiteModelModule, ], providers: [...databaseConfigInitProvider], exports: [ @@ -42,6 +44,8 @@ import { TagModelModule } from './tags/tag-model.module'; CronSubscriptionModelModule, SecretsModelModule, CustomJobTemplateModelModule, + WebsiteModelModule, + DomainModelModule, ], }) export class DatalayerModule {} diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.spec.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.spec.ts index 2251289dd..d3f7197dc 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.spec.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.spec.ts @@ -18,7 +18,7 @@ describe('Finding utils', () => { it('Port findings key', () => { // Arrange // Act - const correlationId = CorrelationKeyUtils.portCorrelationKey( + const correlationKey = CorrelationKeyUtils.portCorrelationKey( '507f1f77bcf86cd799439011', '1.2.3.4', 443, @@ -26,7 +26,7 @@ describe('Finding utils', () => { ); // Assert - expect(correlationId).toBe( + expect(correlationKey).toBe( `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp`, ); }); @@ -34,17 +34,82 @@ describe('Finding utils', () => { it('Domain findings key', () => { // Arrange // Act - const correlationId = CorrelationKeyUtils.domainCorrelationKey( + const correlationKey = CorrelationKeyUtils.domainCorrelationKey( '507f1f77bcf86cd799439011', 'www.stalker.is', ); // Assert - expect(correlationId).toBe( + expect(correlationKey).toBe( `project:507f1f77bcf86cd799439011;domain:www.stalker.is`, ); }); + it('Ip range findings key', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.ipRangeCorrelationKey( + '507f1f77bcf86cd799439011', + '1.2.3.4', + 24, + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;mask:24`, + ); + }); + + it('Website findings key', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + '507f1f77bcf86cd799439011', + '1.2.3.4', + 443, + 'example.com', + '/example/', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp;domain:example.com;path:/example/`, + ); + }); + + it('Website findings key, no domain', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + '507f1f77bcf86cd799439011', + '1.2.3.4', + 443, + '', + '/example/', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp;domain:;path:/example/`, + ); + }); + + it('Website findings key, no path', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + '507f1f77bcf86cd799439011', + '1.2.3.4', + 443, + 'example.com', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp;domain:example.com;path:/`, + ); + }); + describe('Key to service name mapping', () => { it('Host findings key', () => { // Arrange @@ -89,5 +154,178 @@ describe('Finding utils', () => { // Assert expect(name).toBe('DomainsService'); }); + + it('Website findings key', () => { + // Arrange + const correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + '507f1f77bcf86cd799439011', + '1.2.3.4', + 443, + 'example.com', + '/example/', + ); + + // Act + const name = CorrelationKeyUtils.getResourceServiceName(correlationKey); + + // Assert + expect(name).toBe('WebsiteService'); + }); + + it('Website findings key, no domain', () => { + // Arrange + const correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + '507f1f77bcf86cd799439011', + '1.2.3.4', + 443, + '', + '/example/', + ); + + // Act + const name = CorrelationKeyUtils.getResourceServiceName(correlationKey); + + // Assert + expect(name).toBe('WebsiteService'); + }); + + it('Website findings key, no path', () => { + // Arrange + const correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + '507f1f77bcf86cd799439011', + '1.2.3.4', + 443, + 'example.com', + ); + + // Act + const name = CorrelationKeyUtils.getResourceServiceName(correlationKey); + + // Assert + expect(name).toBe('WebsiteService'); + }); + }); + + describe('Generic key generation', () => { + it('Host findings key', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.generateCorrelationKey( + '507f1f77bcf86cd799439011', + null, + '1.2.3.4', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4`, + ); + }); + + it('Port findings key', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.generateCorrelationKey( + '507f1f77bcf86cd799439011', + null, + '1.2.3.4', + 443, + 'tcp', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp`, + ); + }); + + it('Domain findings key', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.generateCorrelationKey( + '507f1f77bcf86cd799439011', + 'www.stalker.is', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;domain:www.stalker.is`, + ); + }); + + it('Ip range findings key', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.generateCorrelationKey( + '507f1f77bcf86cd799439011', + null, + '1.2.3.4', + null, + null, + 24, + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;mask:24`, + ); + }); + + it('Website findings key', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.generateCorrelationKey( + '507f1f77bcf86cd799439011', + 'example.com', + '1.2.3.4', + 443, + 'tcp', + null, + '/example/', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp;domain:example.com;path:/example/`, + ); + }); + + it('Website findings key, no domain', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.generateCorrelationKey( + '507f1f77bcf86cd799439011', + '', + '1.2.3.4', + 443, + 'tcp', + null, + '/example/', + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp;domain:;path:/example/`, + ); + }); + + it('Website findings key, no path', () => { + // Arrange + // Act + const correlationKey = CorrelationKeyUtils.generateCorrelationKey( + '507f1f77bcf86cd799439011', + 'example.com', + '1.2.3.4', + 443, + 'tcp', + null, + null, + ); + + // Assert + expect(correlationKey).toBe( + `project:507f1f77bcf86cd799439011;host:1.2.3.4;port:443;protocol:tcp;domain:example.com;path:/`, + ); + }); }); }); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.ts index 74855f369..27101a74d 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/correlation.utils.ts @@ -47,6 +47,39 @@ export class CorrelationKeyUtils { ); } + public static websiteCorrelationKey( + projectId: string, + ip: string, + port: number, + domainName: string = '', + path: string = '/', + ) { + return CorrelationKeyUtils.buildCorrelationKey( + CorrelationKeyUtils.portCorrelationKey(projectId, ip, port, 'tcp'), + `domain:${domainName}`, + `path:${path}`, + ); + } + + /** + * Valid combinations are: + * + * - domain correlation key: [domainName] + * - host correlation key: [ip] + * - ip range correlation key: [ip,mask] + * - port correlation key: [ip,port,protocol] + * - website correlation key: [ip,port,domainName,path] + * + * For a website correlation key, the domain name can be an empty string + * @param projectId + * @param domainName + * @param ip + * @param port + * @param protocol + * @param mask + * @param path For a website. Should always start with a / + * @returns A correlation key that corresponds to the resource + */ public static generateCorrelationKey( projectId: string, domainName?: string, @@ -54,6 +87,7 @@ export class CorrelationKeyUtils { port?: number, protocol?: string, mask?: number, + path?: string, ): string { if (!projectId) { throw new HttpBadRequestException( @@ -63,15 +97,27 @@ export class CorrelationKeyUtils { let correlationKey = null; const ambiguousRequest = - 'Ambiguous request; must provide a domainName, an ip, an ip and mask or a combination of ip, port and protocol.'; - if (domainName) { - if (ip || port || protocol) - throw new HttpBadRequestException(ambiguousRequest); + 'Ambiguous request for correlation key; Valid combinations are: [domainName], [ip], [ip,mask], [ip,port,protocol] and [ip,port,domainName,path]'; + if (domainName || domainName === '') { + if (ip || port || protocol) { + if (!ip || !port) throw new HttpBadRequestException(ambiguousRequest); - correlationKey = CorrelationKeyUtils.domainCorrelationKey( - projectId, - domainName, - ); + path = path ? path : '/'; + domainName = domainName ?? ''; + + correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + projectId, + ip, + port, + domainName, + path, + ); + } else if (domainName) { + correlationKey = CorrelationKeyUtils.domainCorrelationKey( + projectId, + domainName, + ); + } } else if (ip) { if (port && protocol && !mask) { correlationKey = CorrelationKeyUtils.portCorrelationKey( @@ -111,16 +157,26 @@ export class CorrelationKeyUtils { */ public static getResourceServiceName( correlationKey: string, - ): 'PortService' | 'DomainsService' | 'HostService' | null { + ): + | 'PortService' + | 'DomainsService' + | 'HostService' + | 'WebsiteService' + | null { // Host match if (correlationKey.match(/^project\:[a-f0-9]{24}\;host\:.+/)?.length > 0) { // Port match if ( - correlationKey.match(/.+\;port\:\d{1,5}\;protocol:(tcp|udp)$/)?.length > - 0 - ) - return 'PortService'; - else return 'HostService'; + correlationKey.match(/.+\;port\:\d{1,5}\;protocol\:(tcp|udp)(\;.+)?$/) + ?.length > 0 + ) { + // Website match + if (correlationKey.match(/.+\;domain\:.*\;path\:\/.*$/)?.length > 0) { + return 'WebsiteService'; + } else { + return 'PortService'; + } + } else return 'HostService'; } // Domain match diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/domain/domain.summary.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/domain/domain.summary.ts index f480e59fd..c3e7c81cc 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/domain/domain.summary.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/domain/domain.summary.ts @@ -1,6 +1,15 @@ +import { Prop } from '@nestjs/mongoose'; import { Types } from 'mongoose'; export interface DomainSummary { id: Types.ObjectId; name: string; } + +export class DomainSummaryType implements DomainSummary { + @Prop() + id: Types.ObjectId; + + @Prop() + name: string; +} diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.model.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.model.ts index 0da8d12e9..d35b7892f 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.model.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.model.ts @@ -11,7 +11,7 @@ export class Port { public host: HostSummary; @Prop() - public projectId?: Types.ObjectId; + public projectId: Types.ObjectId; /** * A pseudo-unique key identifying this entity. Used for findings. diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.summary.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.summary.ts new file mode 100644 index 000000000..7950f6059 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.summary.ts @@ -0,0 +1,15 @@ +import { Prop } from '@nestjs/mongoose'; +import { Types } from 'mongoose'; + +export interface PortSummary { + id: Types.ObjectId; + port: number; +} + +export class PortSummaryType implements PortSummary { + @Prop() + id: Types.ObjectId; + + @Prop() + port: number; +} diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/project.module.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/project.module.ts index 872d24890..ddfc3924b 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/project.module.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/project.module.ts @@ -10,6 +10,7 @@ import { HostModule } from './host/host.module'; import { PortModule } from './port/port.module'; import { ProjectController } from './project.controller'; import { ProjectService } from './project.service'; +import { WebsiteModule } from './websites/website.module'; @Module({ imports: [ @@ -22,6 +23,7 @@ import { ProjectService } from './project.service'; PortModule, ConfigModule, SecretsModule, + WebsiteModule, ], controllers: [ProjectController], providers: [ProjectService], diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/project.service.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/project.service.ts index 01805b8a3..aa0ab65b1 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/project.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/project.service.ts @@ -14,6 +14,7 @@ import { HostService } from './host/host.service'; import { PortService } from './port/port.service'; import { CreateProjectDto } from './project.dto'; import { Project, ProjectDocument } from './project.model'; +import { WebsiteService } from './websites/website.service'; @Injectable() export class ProjectService { @@ -29,6 +30,7 @@ export class ProjectService { private readonly findingModel: Model, private readonly portsService: PortService, private readonly secretsService: SecretsService, + private readonly websiteService: WebsiteService, ) {} public async getAll( @@ -97,6 +99,7 @@ export class ProjectService { await this.jobsService.deleteAllForProject(id); await this.subscriptionsService.deleteAllForProject(id); await this.portsService.deleteAllForProject(id); + await this.websiteService.deleteAllForProject(id); await this.findingModel.deleteMany({ projectId: { $eq: new Types.ObjectId(id) }, }); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts new file mode 100644 index 000000000..13f04b5da --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts @@ -0,0 +1,16 @@ +export class WebsitePagingModel { + page: string; + pageSize: string; +} + +export class WebsiteFilterModel { + domain?: Array; + tags?: Array; + project?: Array; + host?: Array; + port?: Array; + firstSeenStartDate?: number; + firstSeenEndDate?: number; + blocked?: boolean; + merged?: boolean; +} diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-model.module.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-model.module.ts new file mode 100644 index 000000000..1a2d8e963 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-model.module.ts @@ -0,0 +1,9 @@ +import { MongooseModule } from '@nestjs/mongoose'; +import { WebsiteSchema } from './website.model'; + +export const WebsiteModelModule = MongooseModule.forFeature([ + { + name: 'websites', + schema: WebsiteSchema, + }, +]); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts new file mode 100644 index 000000000..c330d5cbb --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts @@ -0,0 +1,7 @@ +import { Controller } from '@nestjs/common'; +import { WebsiteService } from './website.service'; + +@Controller('websites') +export class WebsiteController { + constructor(private readonly portsService: WebsiteService) {} +} diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.dto.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.dto.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts new file mode 100644 index 000000000..27fcdcdad --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts @@ -0,0 +1,171 @@ +import { HttpStatus, INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test, TestingModule } from '@nestjs/testing'; +import { randomUUID } from 'crypto'; +import { + TestingData, + checkAuthorizations, + cleanup, + createDomain as createDomains, + createProject, + deleteReq, + getReq, + initTesting, + patchReq, + postReq, + putReq, +} from '../../../../test/e2e.utils'; +import { AppModule } from '../../../app.module'; +import { Role } from '../../../auth/constants'; +import { WebsiteService } from './website.service'; + +describe('Website Controller (e2e)', () => { + let app: INestApplication; + let testData: TestingData; + + let moduleFixture: TestingModule; + let portsService: WebsiteService; + const testPrefix = 'website-controller-e2e-'; + + beforeAll(async () => { + moduleFixture = await Test.createTestingModule({ + imports: [AppModule], + }).compile(); + portsService = await moduleFixture.resolve(WebsiteService); + + app = moduleFixture.createNestApplication(); + app.useGlobalPipes( + new ValidationPipe({ + whitelist: true, + forbidNonWhitelisted: true, + }), + ); + await app.init(); + }); + + beforeEach(async () => { + testData = await initTesting(app); + await cleanup(); + }); + + afterAll(async () => { + await app.close(); + }); + + describe("GET a host's ports /ports", () => { + it('Should get the TCP ports of a host without ports (GET /ports/)', async () => { + // Arrange + const project = await createProject(app, testData, getName()); + const domain = 'www.example.org'; + await createDomains(app, testData, project._id, [domain]); + const rHost = await postReq(app, testData.admin.token, `/hosts`, { + ips: ['192.168.2.1'], + projectId: project._id.toString(), + }); + + const hostId = rHost.body[0]._id; + + // Act + const r = await getReq( + app, + testData.admin.token, + `/ports/?hostId=${hostId}&page=0&pageSize=10&protocol=tcp`, + ); + + // Assert + expect(r.statusCode).toBe(HttpStatus.OK); + expect(r.body.items.length).toStrictEqual(0); + expect(r.body.totalRecords).toStrictEqual(0); + }); + }); + + it('Should have proper authorizations (GET /ports/:id)', async () => { + const success = await checkAuthorizations( + testData, + Role.ReadOnly, + async (givenToken) => { + return await getReq(app, givenToken, `/ports/6450827d0ae00198f250672d`); + }, + ); + expect(success).toBe(true); + }); + + it('Should have proper authorizations (PUT /ports/:id/tags)', async () => { + const success = await checkAuthorizations( + testData, + Role.User, + async (givenToken) => { + return await putReq( + app, + givenToken, + `/ports/6450827d0ae00198f250672d/tags`, + {}, + ); + }, + ); + expect(success).toBe(true); + }); + + it('Should have proper authorizations (GET /ports)', async () => { + // Arrange + const project = await createProject(app, testData, getName()); + const domain = 'www.example.org'; + await createDomains(app, testData, project._id, [domain]); + const rHost = await postReq(app, testData.admin.token, `/hosts`, { + ips: ['192.168.2.1'], + projectId: project._id.toString(), + }); + + const hostId = rHost.body[0]._id; + + const success = await checkAuthorizations( + testData, + Role.ReadOnly, + async (givenToken) => { + return await getReq(app, givenToken, `/ports?`); + }, + ); + expect(success).toBe(true); + }); + + it('Should have proper authorizations (DELETE /ports/:id)', async () => { + const success = await checkAuthorizations( + testData, + Role.User, + async (givenToken) => { + return await deleteReq( + app, + givenToken, + `/ports/6450827d0ae00198f250672d`, + {}, + ); + }, + ); + expect(success).toBe(true); + }); + + it('Should have proper authorizations (DELETE /ports/)', async () => { + const success = await checkAuthorizations( + testData, + Role.User, + async (givenToken) => { + return await deleteReq(app, givenToken, `/ports/`, {}); + }, + ); + expect(success).toBe(true); + }); + + it('Should have proper authorizations (PATCH /ports/)', async () => { + const success = await checkAuthorizations( + testData, + Role.User, + async (givenToken) => { + return await patchReq(app, givenToken, `/ports/`, {}); + }, + ); + expect(success).toBe(true); + }); + + function getName() { + return `${testPrefix}-${randomUUID()}`; + } +}); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts new file mode 100644 index 000000000..4ead5d946 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts @@ -0,0 +1,77 @@ +import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose'; +import { Document, Types } from 'mongoose'; +import { MONGO_TIMESTAMP_SCHEMA_CONFIG } from '../../database.constants'; +import { DomainSummary, DomainSummaryType } from '../domain/domain.summary'; +import { HostSummary, HostSummaryType } from '../host/host.summary'; +import { PortSummary, PortSummaryType } from '../port/port.summary'; + +export type WebsiteDocument = Website & Document; + +@Schema(MONGO_TIMESTAMP_SCHEMA_CONFIG) +export class Website { + @Prop({ type: HostSummaryType }) + public host: HostSummary; + + @Prop({ type: Array }) + public alternativeHosts?: HostSummary[]; + + @Prop({ type: DomainSummaryType }) + public domain?: DomainSummary; + + @Prop({ type: Array }) + public alternativeDomains?: DomainSummary[]; + + @Prop({ type: PortSummaryType }) + public port: PortSummary; + + @Prop({ type: Array }) + public alternativePorts?: PortSummary[]; + + @Prop() + public path: string; + + @Prop() + public sitemap: string[]; + + @Prop() + public previewImage: string; + + @Prop() + public projectId?: Types.ObjectId; + + /** + * A pseudo-unique key identifying this entity. Used for findings. + * This key should not change if this entity were to be recreated. + */ + @Prop() + public correlationKey!: string; + + @Prop() + public tags?: Types.ObjectId[]; + + @Prop() + public updatedAt: number; + + @Prop() + public createdAt: number; + + @Prop() + public lastSeen: number; + + @Prop() + public blocked?: boolean; + + @Prop() + public blockedAt?: number; + + @Prop() + public mergedInId?: Types.ObjectId; +} + +export const WebsiteSchema = SchemaFactory.createForClass(Website); + +// The project id and host id are not included here as the port id - host id - projectId combination is already unique +WebsiteSchema.index( + { 'port.id': 1, 'domain.id': 1, path: 1 }, + { unique: true }, +); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.module.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.module.ts new file mode 100644 index 000000000..2e946fb4f --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { QueueModule } from '../../../job-queue/queue.module'; +import { DatalayerModule } from '../../datalayer.module'; +import { TagsModule } from '../../tags/tag.module'; +import { WebsiteController } from './website.controller'; +import { WebsiteService } from './website.service'; + +@Module({ + imports: [DatalayerModule, TagsModule, QueueModule], + controllers: [WebsiteController], + providers: [WebsiteService], + exports: [WebsiteService], +}) +export class WebsiteModule {} diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts new file mode 100644 index 000000000..238a6f839 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts @@ -0,0 +1,204 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getName } from '../../../../test/test.utils'; +import { AppModule } from '../../../app.module'; +import { TagsService } from '../../tags/tag.service'; +import { DomainsService } from '../domain/domain.service'; +import { HostService } from '../host/host.service'; +import { CreateProjectDto } from '../project.dto'; +import { ProjectService } from '../project.service'; + +import { PortService } from '../port/port.service'; +import { WebsiteService } from './website.service'; + +describe('Website Service', () => { + let moduleFixture: TestingModule; + let hostService: HostService; + let domainService: DomainsService; + let projectService: ProjectService; + let tagsService: TagsService; + let portService: PortService; + let websiteService: WebsiteService; + const testPrefix = 'website-service-ut'; + + beforeAll(async () => { + moduleFixture = await Test.createTestingModule({ + imports: [AppModule], + }).compile(); + hostService = moduleFixture.get(HostService); + domainService = moduleFixture.get(DomainsService); + projectService = moduleFixture.get(ProjectService); + tagsService = moduleFixture.get(TagsService); + portService = moduleFixture.get(PortService); + websiteService = moduleFixture.get(WebsiteService); + }); + + beforeEach(async () => { + const allProjects = await projectService.getAll(); + for (const c of allProjects) { + await projectService.delete(c._id); + } + const tags = await tagsService.getAll(); + for (const t of tags) { + tagsService.delete(t._id); + } + }); + + afterAll(async () => { + await moduleFixture.close(); + }); + + describe('Add websites', () => { + it('Should create a website for a host without domain or path', async () => { + // Arrange + const portNumber = 22; + const c = await project(); + const h = await host('1.1.1.1', c._id.toString()); + const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + + // Act + const w1 = await websiteService.addWebsite( + c._id.toString(), + h[0].ip, + portNumber, + ); + + // Assert + expect(w1._id).toBeTruthy(); + expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.path).toStrictEqual('/'); + expect(w1.domain).toBeNull(); + }); + + it('Should create a website for a host with domain without path', async () => { + // Arrange + const portNumber = 22; + const domainName = 'example.com'; + const c = await project(); + const h = await host('1.1.1.1', c._id.toString()); + const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const d = await domain(domainName, c._id.toString()); + + // Act + const w1 = await websiteService.addWebsite( + c._id.toString(), + h[0].ip, + portNumber, + domainName, + ); + + // Assert + expect(w1._id).toBeTruthy(); + expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.path).toStrictEqual('/'); + expect(w1.domain.name).toStrictEqual(domainName); + }); + + it('Should create a website for a host with domain and path', async () => { + // Arrange + const portNumber = 22; + const domainName = 'example.com'; + const path = '/example/asdf/'; + const c = await project(); + const h = await host('1.1.1.1', c._id.toString()); + const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const d = await domain(domainName, c._id.toString()); + + // Act + const w1 = await websiteService.addWebsite( + c._id.toString(), + h[0].ip, + portNumber, + domainName, + path, + ); + + // Assert + expect(w1._id).toBeTruthy(); + expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.path).toStrictEqual(path); + expect(w1.domain.name).toStrictEqual(domainName); + }); + + it('Should create a website for a host without domain, but with path', async () => { + // Arrange + const portNumber = 22; + const path = '/example/asdf/'; + const c = await project(); + const h = await host('1.1.1.1', c._id.toString()); + const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + + // Act + const w1 = await websiteService.addWebsite( + c._id.toString(), + h[0].ip, + portNumber, + '', + path, + ); + + // Assert + expect(w1._id).toBeTruthy(); + expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.path).toStrictEqual(path); + expect(w1.domain).toBeNull(); + }); + + it('Should not create a second website for twice the same host-port-domain-project', async () => { + // Arrange + const portNumber = 22; + const path = '/example/asdf/'; + const domainName = 'example.com'; + const c = await project(); + const h = await host('1.1.1.1', c._id.toString()); + const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const d = await domain(domainName, c._id.toString()); + const w1 = await websiteService.addWebsite( + c._id.toString(), + h[0].ip, + portNumber, + domainName, + path, + ); + + // Act + const w2 = await websiteService.addWebsite( + c._id.toString(), + h[0].ip, + portNumber, + domainName, + path, + ); + + // Assert + const allWebsites = await websiteService.getAll(); + expect(allWebsites.length).toStrictEqual(1); + expect(allWebsites[0]._id.toString()).toStrictEqual(w1._id.toString()); + }); + }); + + async function project(name: string = '') { + const ccDto: CreateProjectDto = { name: `${getName(testPrefix)}` }; + return await projectService.addProject(ccDto); + } + + async function host(ip: string, projectId: string) { + return await hostService.addHosts([ip], projectId); + } + + async function domain(name: string, projectId: string) { + return await domainService.addDomain(name, projectId); + } + + async function port( + port: number, + hostId: string, + projectId: string, + protocol: 'tcp' | 'udp' = 'tcp', + ) { + return await portService.addPort(hostId, projectId, port, protocol); + } +}); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts new file mode 100644 index 000000000..1c54c23e7 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts @@ -0,0 +1,232 @@ +import { Injectable, Logger, NotImplementedException } from '@nestjs/common'; +import { InjectModel } from '@nestjs/mongoose'; +import { DeleteResult } from 'mongodb'; +import { FilterQuery, Model, Types } from 'mongoose'; +import { HttpNotFoundException } from '../../../../exceptions/http.exceptions'; +import { WebsiteFinding } from '../../../findings/findings.service'; +import { FindingsQueue } from '../../../job-queue/findings-queue'; +import { TagsService } from '../../tags/tag.service'; +import { CorrelationKeyUtils } from '../correlation.utils'; +import { Domain } from '../domain/domain.model'; +import { DomainSummary } from '../domain/domain.summary'; +import { Host } from '../host/host.model'; +import { Port } from '../port/port.model'; +import { WebsiteFilterModel } from './website-filter.model'; +import { Website, WebsiteDocument } from './website.model'; + +@Injectable() +export class WebsiteService { + private logger = new Logger(WebsiteService.name); + + constructor( + @InjectModel('websites') private readonly websiteModel: Model, + @InjectModel('domain') private readonly domainModel: Model, + @InjectModel('host') private readonly hostModel: Model, + @InjectModel('port') private readonly portModel: Model, + private findingsQueue: FindingsQueue, + private tagsService: TagsService, + ) {} + + public async addWebsite( + projectId: string, + ip: string, + port: number, + domain: string = undefined, + path: string = '/', + ) { + const projectIdObj = new Types.ObjectId(projectId); + const existingPort = await this.portModel.findOne({ + 'host.ip': { $eq: ip }, + port: { $eq: port }, + projectId: { $eq: projectIdObj }, + layer4Protocol: { $eq: 'tcp' }, + }); + + if (!existingPort) { + this.logger.error( + `Failed to add a website because the port did not exist (project: ${projectId}, IP: ${ip}, port: ${port})`, + ); + throw new HttpNotFoundException(); + } + + // Validate the domain + let existingDomainSummary: DomainSummary = undefined; + if (domain) { + const existingDomain = await this.domainModel.findOne( + { + projectId: { $eq: projectIdObj }, + name: { $eq: domain }, + }, + '_id name', + ); + + if (existingDomain) { + existingDomainSummary = { + id: existingDomain._id, + name: existingDomain.name, + }; + } else { + this.logger.error( + `Domain not found while adding a website (project: ${projectId}, domain: ${domain})`, + ); + throw new HttpNotFoundException(); + } + } else { + existingDomainSummary = undefined; + } + + // Search for a website with the proper port id, domain and path. + // domain may or may not exist, it depends if it was found in the domains collection + const searchQuery: FilterQuery = { + 'port.id': { $eq: existingPort._id }, + 'domain.id': { + $eq: existingDomainSummary ? existingDomainSummary.id : null, + }, + path: { $eq: path }, + }; + + return this.websiteModel.findOneAndUpdate( + searchQuery, + { + $set: { lastSeen: Date.now() }, + $setOnInsert: { + host: existingPort.host, + domain: existingDomainSummary ?? null, + port: { id: existingPort._id, port: existingPort.port }, + path: path, + sitemap: ['/'], + projectId: projectIdObj, + correlationKey: CorrelationKeyUtils.websiteCorrelationKey( + projectId, + ip, + port, + existingDomainSummary ? existingDomainSummary.name : '', + path, + ), + }, + }, + { upsert: true, new: true }, + ); + } + + /** + * This method creates a WebsiteFinding for each domain linked to the host, + * as well as one for the host's direct ip port access, without a domain. + * @param jobId + * @param projectId + * @param ip + * @param port + * @param path + * @returns + */ + public async emitWebsiteFindingsForAllHostDomains( + jobId: string, + projectId: string, + ip: string, + port: number, + path: string = '/', + ) { + const host = await this.hostModel.findOne({ + ip: { $eq: ip }, + projectId: { $eq: new Types.ObjectId(projectId) }, + }); + + if (!host) { + this.logger.error( + `Failed to add the websites because the host did not exist (project: ${projectId}, IP: ${ip})`, + ); + throw new HttpNotFoundException(); + } + + const websiteFindingBase: Omit = { + type: 'WebsiteFinding', + key: 'WebsiteFinding', + ip: ip, + path: path, + port: port, + fields: [], + }; + + // We will create a website finding + let findings: WebsiteFinding[] = [ + { + domain: '', + ...websiteFindingBase, + }, + ]; + + for (const domainSummary of host.domains) { + findings.push({ + domain: domainSummary.name, + ...websiteFindingBase, + }); + + if (findings.length >= 10) { + await this.findingsQueue.publish(...findings); + findings = []; + } + } + + this.findingsQueue.publishForJob(jobId, ...findings); + } + + public async deleteAllForProject(projectId: string): Promise { + return await this.websiteModel.deleteMany({ + projectId: { $eq: new Types.ObjectId(projectId) }, + }); + } + + public async delete(websiteId: string): Promise { + return await this.websiteModel.deleteOne({ + _id: { $eq: new Types.ObjectId(websiteId) }, + }); + } + + public async get(websiteId: string) { + return await this.websiteModel.findById(websiteId); + } + + public async getAll( + page: number = null, + pageSize: number = null, + filter: WebsiteFilterModel = null, + ): Promise { + let query; + if (filter) { + query = this.websiteModel.find(this.buildFilters(filter)); + } else { + query = this.websiteModel.find({}); + } + + if (page != null && pageSize != null) { + query = query.skip(page * pageSize).limit(pageSize); + } + return await query; + } + + private async buildFilters(filter: WebsiteFilterModel) { + throw new NotImplementedException(); + } + + public async keyIsBlocked(correlationKey: string) { + const website = await this.websiteModel.findOne( + { correlationKey: { $eq: correlationKey } }, + 'blocked mergedInId', + ); + return website && (!!website.mergedInId || website.blocked); + } + + /** + * TODO: + * + * When a host is deleted: + * delete the websites for the host + * unlink the host if in alternative hosts + * When a domain is deleted: + * delete the websites for the domain + * unlink the domain if in alternative domains + * When a port is deleted: + * delete the websites for the port + * unlink the port if in alternative ports + */ +} diff --git a/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.module.ts b/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.module.ts index 994d38723..e8faadfcd 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.module.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.module.ts @@ -3,6 +3,7 @@ import { MongooseModule } from '@nestjs/mongoose'; import { DomainsModule } from '../../reporting/domain/domain.module'; import { HostModule } from '../../reporting/host/host.module'; import { PortModule } from '../../reporting/port/port.module'; +import { WebsiteModule } from '../../reporting/websites/website.module'; import { SubscriptionTriggersController } from './subscription-triggers.controller'; import { SubscriptionTriggerSchema } from './subscription-triggers.model'; import { SubscriptionTriggersService } from './subscription-triggers.service'; @@ -18,6 +19,7 @@ import { SubscriptionTriggersService } from './subscription-triggers.service'; HostModule, DomainsModule, PortModule, + WebsiteModule, ], controllers: [SubscriptionTriggersController], providers: [SubscriptionTriggersService], diff --git a/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.service.ts b/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.service.ts index 66f36e385..f052ba3b3 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscription-triggers/subscription-triggers.service.ts @@ -6,6 +6,7 @@ import { CorrelationKeyUtils } from '../../reporting/correlation.utils'; import { DomainsService } from '../../reporting/domain/domain.service'; import { HostService } from '../../reporting/host/host.service'; import { PortService } from '../../reporting/port/port.service'; +import { WebsiteService } from '../../reporting/websites/website.service'; import { SubscriptionTrigger, SubscriptionTriggerDocument, @@ -21,6 +22,7 @@ export class SubscriptionTriggersService { private readonly hostsService: HostService, private readonly domainsService: DomainsService, private readonly portsService: PortService, + private readonly websiteService: WebsiteService, ) {} public async isTriggerBlocked(correlationKey: string): Promise { @@ -34,6 +36,8 @@ export class SubscriptionTriggersService { return await this.domainsService.keyIsBlocked(correlationKey); case 'HostService': return await this.hostsService.keyIsBlocked(correlationKey); + case 'WebsiteService': + return await this.websiteService.keyIsBlocked(correlationKey); default: return false; } diff --git a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts index 4916c2797..490e6bda8 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts @@ -5,6 +5,7 @@ import { ConfigService } from '../../../database/admin/config/config.service'; import { CustomJobsService } from '../../../database/custom-jobs/custom-jobs.service'; import { JobsService } from '../../../database/jobs/jobs.service'; import { PortService } from '../../../database/reporting/port/port.service'; +import { WebsiteService } from '../../../database/reporting/websites/website.service'; import { SecretsService } from '../../../database/secrets/secrets.service'; import { EventSubscriptionsService } from '../../../database/subscriptions/event-subscriptions/event-subscriptions.service'; import { SubscriptionTriggersService } from '../../../database/subscriptions/subscription-triggers/subscription-triggers.service'; @@ -25,6 +26,7 @@ export class CustomFindingHandler extends JobFindingHandlerBase { + protected logger: Logger = new Logger('WebsiteHandler'); + + constructor( + private websiteService: WebsiteService, + jobService: JobsService, + subscriptionsService: EventSubscriptionsService, + customJobsService: CustomJobsService, + configService: ConfigService, + subscriptionTriggersService: SubscriptionTriggersService, + secretsService: SecretsService, + ) { + super( + jobService, + subscriptionsService, + customJobsService, + configService, + subscriptionTriggersService, + secretsService, + ); + } + + protected async executeCore(command: WebsiteCommand) { + await this.websiteService.addWebsite( + command.projectId, + command.finding.ip, + command.finding.port, + command.finding.domain, + command.finding.path, + ); + } +} diff --git a/packages/backend/jobs-manager/service/src/modules/findings/commands/findings-commands.ts b/packages/backend/jobs-manager/service/src/modules/findings/commands/findings-commands.ts index 3848db88f..ca3354fc8 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/commands/findings-commands.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/commands/findings-commands.ts @@ -8,6 +8,8 @@ import { HostnameIpCommand } from './JobFindings/hostname-ip.command'; import { HostnameIpHandler } from './JobFindings/hostname-ip.handler'; import { PortCommand } from './JobFindings/port.command'; import { PortHandler } from './JobFindings/port.handler'; +import { WebsiteCommand } from './JobFindings/website.command'; +import { WebsiteHandler } from './JobFindings/website.handler'; export const FindingsCommandMapping = [ { @@ -30,6 +32,11 @@ export const FindingsCommandMapping = [ handler: PortHandler, command: PortCommand, }, + { + finding: 'WebsiteFinding', + handler: WebsiteHandler, + command: WebsiteCommand, + }, { finding: 'CustomFinding', handler: CustomFindingHandler, diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.module.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.module.ts index e0dcdf2af..7455b7259 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.module.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.module.ts @@ -11,6 +11,7 @@ import { DomainsModule } from '../database/reporting/domain/domain.module'; import { HostModule } from '../database/reporting/host/host.module'; import { PortModule } from '../database/reporting/port/port.module'; import { ProjectModule } from '../database/reporting/project.module'; +import { WebsiteModule } from '../database/reporting/websites/website.module'; import { SecretsModule } from '../database/secrets/secrets.module'; import { EventSubscriptionsModule } from '../database/subscriptions/event-subscriptions/event-subscriptions.module'; import { SubscriptionTriggersModule } from '../database/subscriptions/subscription-triggers/subscription-triggers.module'; @@ -35,6 +36,7 @@ import { JobLogsConsumer } from './job-logs.consumer'; ConfigModule, SubscriptionTriggersModule, SecretsModule, + WebsiteModule, ], controllers: [FindingsController], providers: [FindingsService, ...FindingsHandlers], diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts index 489a6a31f..86d754667 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts @@ -130,7 +130,6 @@ describe('Findings Service Spec', () => { [undefined, undefined, undefined], ['example.org', undefined, 80], ['example.org', '1.1.1.1', undefined], - ['example.org', '1.1.1.1', 80], [undefined, undefined, 1], ])( 'Save - Invalid finding correlation information - Throws', diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts index cd26c30c4..ac51125f4 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts @@ -18,6 +18,7 @@ import { IpRangeCommand } from './commands/Findings/ipRange.command'; import { CustomFindingCommand } from './commands/JobFindings/custom.command'; import { HostnameIpCommand } from './commands/JobFindings/hostname-ip.command'; import { PortCommand } from './commands/JobFindings/port.command'; +import { WebsiteCommand } from './commands/JobFindings/website.command'; import { CustomFindingFieldDto } from './finding.dto'; export type Finding = @@ -26,6 +27,7 @@ export type Finding = | IpFinding | IpRangeFinding | PortFinding + | WebsiteFinding | JobStatusFinding | CreateCustomFinding; @@ -50,6 +52,15 @@ export class PortFinding extends FindingBase { ]; } +export class WebsiteFinding extends FindingBase { + type: 'WebsiteFinding'; + key: 'WebsiteFinding'; + ip: string; + port: number; + domain: string = ''; + path: string = '/'; +} + export class CreateCustomFinding extends FindingBase { type: 'CustomFinding'; key: string; @@ -271,93 +282,116 @@ export class FindingsService { return; } - switch (finding.type) { - case 'HostnameIpFinding': - finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( - projectId, - null, - finding.ip, - ); - this.commandBus.execute( - new HostnameIpCommand( - jobId, + try { + switch (finding.type) { + case 'HostnameIpFinding': + finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( projectId, - HostnameIpCommand.name, - finding, - ), - ); - break; - case 'HostnameFinding': - finding.projectId = finding.projectId ? finding.projectId : projectId; - finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( - finding.projectId, - finding.domainName, - ); - this.commandBus.execute( - new HostnameCommand(finding.projectId, HostnameCommand.name, finding), - ); - break; - - case 'IpFinding': - finding.projectId = finding.projectId ? finding.projectId : projectId; - finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( - finding.projectId, - null, - finding.ip, - ); - this.commandBus.execute( - new IpCommand(finding.projectId, IpCommand.name, finding), - ); - break; - - case 'IpRangeFinding': - finding.projectId = finding.projectId ? finding.projectId : projectId; - finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( - finding.projectId, - null, - finding.ip, - null, - null, - finding.mask, - ); - this.commandBus.execute( - new IpRangeCommand(finding.projectId, IpRangeCommand.name, finding), - ); - break; - - case 'PortFinding': - finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( - projectId, - null, - finding.ip, - finding.port, - finding.fields[0]?.data, - ); - this.commandBus.execute( - new PortCommand(jobId, projectId, PortCommand.name, finding), - ); - break; - - case 'CustomFinding': - finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( - projectId, - finding.domainName, - finding.ip, - finding.port, - finding.protocol, - ); - this.commandBus.execute( - new CustomFindingCommand( - jobId, + null, + finding.ip, + ); + this.commandBus.execute( + new HostnameIpCommand( + jobId, + projectId, + HostnameIpCommand.name, + finding, + ), + ); + break; + case 'HostnameFinding': + finding.projectId = finding.projectId ? finding.projectId : projectId; + finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( + finding.projectId, + finding.domainName, + ); + this.commandBus.execute( + new HostnameCommand( + finding.projectId, + HostnameCommand.name, + finding, + ), + ); + break; + + case 'IpFinding': + finding.projectId = finding.projectId ? finding.projectId : projectId; + finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( + finding.projectId, + null, + finding.ip, + ); + this.commandBus.execute( + new IpCommand(finding.projectId, IpCommand.name, finding), + ); + break; + + case 'IpRangeFinding': + finding.projectId = finding.projectId ? finding.projectId : projectId; + finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( + finding.projectId, + null, + finding.ip, + null, + null, + finding.mask, + ); + this.commandBus.execute( + new IpRangeCommand(finding.projectId, IpRangeCommand.name, finding), + ); + break; + + case 'PortFinding': + finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( projectId, - CustomFindingCommand.name, - finding, - ), - ); - break; + null, + finding.ip, + finding.port, + finding.fields[0]?.data, + ); + this.commandBus.execute( + new PortCommand(jobId, projectId, PortCommand.name, finding), + ); + break; - default: - this.logger.error(`Unknown finding type ${finding['type']}`); + case 'WebsiteFinding': + finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( + projectId, + finding.domain, + finding.ip, + finding.port, + 'tcp', + null, + finding.path, + ); + this.commandBus.execute( + new WebsiteCommand(jobId, projectId, WebsiteCommand.name, finding), + ); + break; + + case 'CustomFinding': + finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( + projectId, + finding.domainName, + finding.ip, + finding.port, + finding.protocol, + ); + this.commandBus.execute( + new CustomFindingCommand( + jobId, + projectId, + CustomFindingCommand.name, + finding, + ), + ); + break; + + default: + this.logger.error(`Unknown finding type ${finding['type']}`); + } + } catch (err) { + this.logger.error(err); } } } diff --git a/packages/backend/jobs-manager/service/src/modules/job-queue/findings-queue.ts b/packages/backend/jobs-manager/service/src/modules/job-queue/findings-queue.ts index e6eeca819..0e3f07ec7 100644 --- a/packages/backend/jobs-manager/service/src/modules/job-queue/findings-queue.ts +++ b/packages/backend/jobs-manager/service/src/modules/job-queue/findings-queue.ts @@ -2,4 +2,9 @@ import { Finding } from '../findings/findings.service'; export abstract class FindingsQueue { public abstract publish(...findings: Finding[]): Promise; + + public abstract publishForJob( + jobId: string, + ...findings: Finding[] + ): Promise; } diff --git a/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts b/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts index 44b230dda..e1cd0ed31 100644 --- a/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts +++ b/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts @@ -1,6 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { Message, Producer } from 'kafkajs'; import { orchestratorConstants } from '../auth/constants'; +import { Finding } from '../findings/findings.service'; import { FindingsQueue } from './findings-queue'; @Injectable() @@ -10,6 +11,13 @@ export class KafkaFindingsQueue implements FindingsQueue { constructor(private producer: Producer) {} public async publish(...findings: any[]) { + this.publishForJob(undefined, ...findings); + } + + public async publishForJob( + jobId?: string, + ...findings: Finding[] + ): Promise { this.logger.debug( `Publishing ${findings.length} findings to the message queue on topic ${ orchestratorConstants.topics.findings @@ -21,6 +29,7 @@ export class KafkaFindingsQueue implements FindingsQueue { key: null, value: JSON.stringify({ FindingsJson: JSON.stringify({ findings: findings }), + JobId: jobId, }), }, ]; diff --git a/packages/backend/jobs-manager/service/src/modules/job-queue/null-findings-queue.ts b/packages/backend/jobs-manager/service/src/modules/job-queue/null-findings-queue.ts index c9f5adb1d..01162c2dd 100644 --- a/packages/backend/jobs-manager/service/src/modules/job-queue/null-findings-queue.ts +++ b/packages/backend/jobs-manager/service/src/modules/job-queue/null-findings-queue.ts @@ -6,6 +6,10 @@ export class NullFindingsQueue implements FindingsQueue { private logger = new Logger(NullFindingsQueue.name); public async publish(...findings: any[]) { + this.publishForJob(undefined, ...findings); + } + + public async publishForJob(...findings: any[]) { this.logger.debug('Findings not posted to findings queue.'); } } From ad4aa3996bdd9e4ba70b6cc3565b70aec49a289e Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 24 Jun 2024 14:56:48 -0400 Subject: [PATCH 02/11] list interface and resource view for websites. Includes blocking, tagging and deleting capabilities --- docs/docs/concepts/findings.md | 5 +- .../stalker_job_sdk/__init__.py | 2 + .../websites/website-filter.model.ts | 6 +- .../reporting/websites/website.controller.ts | 81 ++++- .../reporting/websites/website.dto.ts | 93 ++++++ .../reporting/websites/website.e2e-spec.ts | 89 ++---- .../reporting/websites/website.model.ts | 5 +- .../websites/website.service.spec.ts | 283 ++++++++++++++++- .../reporting/websites/website.service.ts | 173 ++++++++++- .../commands/JobFindings/custom.handler.ts | 1 + .../commands/JobFindings/website.handler.ts | 4 + .../src/modules/findings/findings.service.ts | 1 + .../modules/job-queue/kafka-findings-queue.ts | 1 + .../src/app/api/websites/websites.service.ts | 98 ++++++ .../stalker-app/src/app/app-routing.module.ts | 4 + .../list-websites.component.html | 122 ++++++++ .../list-websites.component.scss | 16 + .../list-websites/list-websites.component.ts | 286 +++++++++++++++++ .../view-website/view-website.component.html | 197 ++++++++++++ .../view-website/view-website.component.scss | 89 ++++++ .../view-website/view-website.component.ts | 292 ++++++++++++++++++ .../websites/websites-interactions.service.ts | 161 ++++++++++ .../app/modules/websites/websites.module.ts | 20 ++ .../components/sidebar/sidebar.component.html | 16 + .../components/sidebar/sidebar.component.ts | 1 + .../app/shared/types/ports/port.summary.ts | 4 + .../app/shared/types/websites/website.type.ts | 27 ++ .../filtered-paginated-table.component.ts | 6 +- 28 files changed, 1989 insertions(+), 94 deletions(-) create mode 100644 packages/frontend/stalker-app/src/app/api/websites/websites.service.ts create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.html create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.scss create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.ts create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/websites-interactions.service.ts create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/websites.module.ts create mode 100644 packages/frontend/stalker-app/src/app/shared/types/ports/port.summary.ts create mode 100644 packages/frontend/stalker-app/src/app/shared/types/websites/website.type.ts diff --git a/docs/docs/concepts/findings.md b/docs/docs/concepts/findings.md index 86139f292..f28199b66 100644 --- a/docs/docs/concepts/findings.md +++ b/docs/docs/concepts/findings.md @@ -228,7 +228,8 @@ Example: "ip": "1.2.3.4", "port": 80, "domain": "example.com", - "path": "/" + "path": "/", + "ssl": false } ``` @@ -240,6 +241,7 @@ port = 80 ip = "1.2.3.4" domain = "example.com" path = "/" +ssl = False log_finding( WebsiteFinding( @@ -248,6 +250,7 @@ log_finding( port, domain, path, + ssl, "New website", [], "WebsiteFinding", diff --git a/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py b/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py index 69e81c8e1..7909512fa 100644 --- a/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py +++ b/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py @@ -82,6 +82,7 @@ def __init__( port: int, domain: str, path: str, + ssl: bool = None, name: str = None, fields: list[Field] = [], type: str = "CustomFinding", @@ -91,6 +92,7 @@ def __init__( self.port = port self.domain = domain self.path = path + self.ssl = ssl class DomainFinding(Finding): def __init__( diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts index 13f04b5da..952c11d1c 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website-filter.model.ts @@ -4,11 +4,11 @@ export class WebsitePagingModel { } export class WebsiteFilterModel { - domain?: Array; + domains?: Array; tags?: Array; project?: Array; - host?: Array; - port?: Array; + hosts?: Array; + ports?: Array; firstSeenStartDate?: number; firstSeenEndDate?: number; blocked?: boolean; diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts index c330d5cbb..51224377a 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.controller.ts @@ -1,7 +1,84 @@ -import { Controller } from '@nestjs/common'; +import { + Body, + Controller, + Delete, + Get, + Param, + Patch, + Put, + Query, + UseGuards, +} from '@nestjs/common'; +import { MongoIdDto } from '../../../../types/dto/mongo-id.dto'; +import { TagItemDto } from '../../../../types/dto/tag-item.dto'; +import { Page } from '../../../../types/page.type'; +import { Role } from '../../../auth/constants'; +import { Roles } from '../../../auth/decorators/roles.decorator'; +import { JwtAuthGuard } from '../../../auth/guards/jwt-auth.guard'; +import { RolesGuard } from '../../../auth/guards/role.guard'; +import { + BatchEditWebsitesDto, + DeleteManyWebsitesDto, + GetWebsitesDto, +} from './website.dto'; +import { WebsiteDocument } from './website.model'; import { WebsiteService } from './website.service'; @Controller('websites') export class WebsiteController { - constructor(private readonly portsService: WebsiteService) {} + constructor(private readonly websiteService: WebsiteService) {} + + @UseGuards(JwtAuthGuard, RolesGuard) + @Roles(Role.ReadOnly) + @Get() + async getWebsites( + @Query() dto: GetWebsitesDto, + ): Promise> { + const totalRecords = await this.websiteService.count(dto); + const items = await this.websiteService.getAll(dto.page, dto.pageSize, dto); + + return { + items, + totalRecords, + }; + } + + @UseGuards(JwtAuthGuard, RolesGuard) + @Roles(Role.User) + @Put(':id/tags') + async tagPort(@Param() idDto: MongoIdDto, @Body() tagDto: TagItemDto) { + return await this.websiteService.tagWebsite( + idDto.id, + tagDto.tagId, + tagDto.isTagged, + ); + } + + @UseGuards(JwtAuthGuard, RolesGuard) + @Roles(Role.ReadOnly) + @Get(':id') + async getWebsite(@Param() idDto: MongoIdDto) { + return await this.websiteService.get(idDto.id); + } + + @UseGuards(JwtAuthGuard, RolesGuard) + @Roles(Role.User) + @Delete() + async deleteWebsites(@Body() dto: DeleteManyWebsitesDto) { + return await this.websiteService.deleteMany(dto.websiteIds); + } + + @UseGuards(JwtAuthGuard, RolesGuard) + @Roles(Role.User) + @Delete(':id') + async deleteWebsite(@Param() idDto: MongoIdDto) { + return await this.websiteService.delete(idDto.id); + } + + @UseGuards(JwtAuthGuard, RolesGuard) + @Roles(Role.User) + @Patch() + async batchEdit(@Body() dto: BatchEditWebsitesDto) { + return await this.websiteService.batchEdit(dto); + } } diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.dto.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.dto.ts index e69de29bb..1be054bea 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.dto.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.dto.ts @@ -0,0 +1,93 @@ +import { Transform, Type } from 'class-transformer'; +import { + IsArray, + IsBoolean, + IsInt, + IsMongoId, + IsOptional, + IsPort, + IsString, + Max, + Min, +} from 'class-validator'; +import { Types } from 'mongoose'; +import { booleanStringToBoolean } from '../../../../utils/boolean-string-to-boolean'; + +export class GetWebsitesDto { + @IsInt() + @Min(0) + @Type(() => Number) + page: number = 0; + + @IsInt() + @Min(1) + @Max(100) + @Type(() => Number) + pageSize: number = 10; + + @IsOptional() + @IsString({ each: true }) + @IsArray() + hosts: string[]; + + @IsOptional() + @IsString({ each: true }) + @IsArray() + domains: string[]; + + @IsOptional() + @IsString({ each: true }) + @IsArray() + paths: string[]; + + @IsOptional() + @IsMongoId({ each: true }) + @IsArray() + project: string[]; + + @IsOptional() + @IsMongoId({ each: true }) + @IsArray() + tags: string[]; + + @IsOptional() + @IsInt() + @Type(() => Number) + firstSeenStartDate: number; + + @IsOptional() + @IsInt() + @Type(() => Number) + firstSeenEndDate: number; + + @IsOptional() + @IsPort({ each: true }) + @IsArray() + ports: number[]; + + @IsOptional() + @IsBoolean() + @Transform(booleanStringToBoolean) + blocked: boolean; + + @IsOptional() + @IsBoolean() + @Transform(booleanStringToBoolean) + merged: boolean; +} + +export class DeleteManyWebsitesDto { + @IsMongoId({ each: true }) + @IsArray() + websiteIds: string[]; +} + +export class BatchEditWebsitesDto { + @IsArray() + @IsMongoId({ each: true }) + websiteIds: Types.ObjectId[]; + + @IsOptional() + @IsBoolean() + block: boolean; +} diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts index 27fcdcdad..4e2bfd64d 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts @@ -1,18 +1,15 @@ -import { HttpStatus, INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; import { randomUUID } from 'crypto'; import { TestingData, checkAuthorizations, cleanup, - createDomain as createDomains, - createProject, deleteReq, getReq, initTesting, patchReq, - postReq, - putReq, + putReq } from '../../../../test/e2e.utils'; import { AppModule } from '../../../app.module'; import { Role } from '../../../auth/constants'; @@ -51,83 +48,51 @@ describe('Website Controller (e2e)', () => { await app.close(); }); - describe("GET a host's ports /ports", () => { - it('Should get the TCP ports of a host without ports (GET /ports/)', async () => { - // Arrange - const project = await createProject(app, testData, getName()); - const domain = 'www.example.org'; - await createDomains(app, testData, project._id, [domain]); - const rHost = await postReq(app, testData.admin.token, `/hosts`, { - ips: ['192.168.2.1'], - projectId: project._id.toString(), - }); - - const hostId = rHost.body[0]._id; - - // Act - const r = await getReq( - app, - testData.admin.token, - `/ports/?hostId=${hostId}&page=0&pageSize=10&protocol=tcp`, - ); - - // Assert - expect(r.statusCode).toBe(HttpStatus.OK); - expect(r.body.items.length).toStrictEqual(0); - expect(r.body.totalRecords).toStrictEqual(0); - }); - }); - - it('Should have proper authorizations (GET /ports/:id)', async () => { + describe('Get websites', () => { + + it('Should have proper authorizations (PUT /websites/:id/tags)', async () => { const success = await checkAuthorizations( testData, - Role.ReadOnly, + Role.User, async (givenToken) => { - return await getReq(app, givenToken, `/ports/6450827d0ae00198f250672d`); + return await putReq( + app, + givenToken, + `/websites/6450827d0ae00198f250672d/tags`, + {}, + ); }, ); expect(success).toBe(true); }); - it('Should have proper authorizations (PUT /ports/:id/tags)', async () => { + it('Should have proper authorizations (GET /websites)', async () => { const success = await checkAuthorizations( testData, - Role.User, + Role.ReadOnly, async (givenToken) => { - return await putReq( - app, - givenToken, - `/ports/6450827d0ae00198f250672d/tags`, - {}, - ); + return await getReq(app, givenToken, `/websites/`); }, ); expect(success).toBe(true); }); - it('Should have proper authorizations (GET /ports)', async () => { - // Arrange - const project = await createProject(app, testData, getName()); - const domain = 'www.example.org'; - await createDomains(app, testData, project._id, [domain]); - const rHost = await postReq(app, testData.admin.token, `/hosts`, { - ips: ['192.168.2.1'], - projectId: project._id.toString(), - }); - - const hostId = rHost.body[0]._id; - + it('Should have proper authorizations (GET /websites/:id)', async () => { const success = await checkAuthorizations( testData, Role.ReadOnly, async (givenToken) => { - return await getReq(app, givenToken, `/ports?`); + return await getReq( + app, + givenToken, + `/websites/6450827d0ae00198f250672d`, + ); }, ); expect(success).toBe(true); }); - it('Should have proper authorizations (DELETE /ports/:id)', async () => { + it('Should have proper authorizations (DELETE /websites/:id)', async () => { const success = await checkAuthorizations( testData, Role.User, @@ -135,7 +100,7 @@ describe('Website Controller (e2e)', () => { return await deleteReq( app, givenToken, - `/ports/6450827d0ae00198f250672d`, + `/websites/6450827d0ae00198f250672d`, {}, ); }, @@ -143,23 +108,23 @@ describe('Website Controller (e2e)', () => { expect(success).toBe(true); }); - it('Should have proper authorizations (DELETE /ports/)', async () => { + it('Should have proper authorizations (DELETE /websites/)', async () => { const success = await checkAuthorizations( testData, Role.User, async (givenToken) => { - return await deleteReq(app, givenToken, `/ports/`, {}); + return await deleteReq(app, givenToken, `/websites/`, {}); }, ); expect(success).toBe(true); }); - it('Should have proper authorizations (PATCH /ports/)', async () => { + it('Should have proper authorizations (PATCH /websites/)', async () => { const success = await checkAuthorizations( testData, Role.User, async (givenToken) => { - return await patchReq(app, givenToken, `/ports/`, {}); + return await patchReq(app, givenToken, `/websites/`, {}); }, ); expect(success).toBe(true); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts index 4ead5d946..ff64cdbc9 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.model.ts @@ -31,9 +31,12 @@ export class Website { public path: string; @Prop() - public sitemap: string[]; + public ssl?: boolean; @Prop() + public sitemap: string[]; + + @Prop({ select: false }) public previewImage: string; @Prop() diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts index 238a6f839..d72ec95f9 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.spec.ts @@ -7,7 +7,14 @@ import { HostService } from '../host/host.service'; import { CreateProjectDto } from '../project.dto'; import { ProjectService } from '../project.service'; +import { getModelToken } from '@nestjs/mongoose'; +import { Model } from 'mongoose'; +import { DomainDocument } from '../domain/domain.model'; +import { HostDocument } from '../host/host.model'; +import { PortDocument } from '../port/port.model'; import { PortService } from '../port/port.service'; +import { WebsiteFilterModel } from './website-filter.model'; +import { Website, WebsiteDocument } from './website.model'; import { WebsiteService } from './website.service'; describe('Website Service', () => { @@ -19,6 +26,7 @@ describe('Website Service', () => { let portService: PortService; let websiteService: WebsiteService; const testPrefix = 'website-service-ut'; + let websiteModel: Model; beforeAll(async () => { moduleFixture = await Test.createTestingModule({ @@ -30,6 +38,7 @@ describe('Website Service', () => { tagsService = moduleFixture.get(TagsService); portService = moduleFixture.get(PortService); websiteService = moduleFixture.get(WebsiteService); + websiteModel = moduleFixture.get>(getModelToken('websites')); }); beforeEach(async () => { @@ -53,19 +62,19 @@ describe('Website Service', () => { const portNumber = 22; const c = await project(); const h = await host('1.1.1.1', c._id.toString()); - const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const p = await port(portNumber, h._id.toString(), c._id.toString()); // Act const w1 = await websiteService.addWebsite( c._id.toString(), - h[0].ip, + h.ip, portNumber, ); // Assert expect(w1._id).toBeTruthy(); expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); - expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h._id.toString()); expect(w1.path).toStrictEqual('/'); expect(w1.domain).toBeNull(); }); @@ -76,13 +85,13 @@ describe('Website Service', () => { const domainName = 'example.com'; const c = await project(); const h = await host('1.1.1.1', c._id.toString()); - const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const p = await port(portNumber, h._id.toString(), c._id.toString()); const d = await domain(domainName, c._id.toString()); // Act const w1 = await websiteService.addWebsite( c._id.toString(), - h[0].ip, + h.ip, portNumber, domainName, ); @@ -90,7 +99,7 @@ describe('Website Service', () => { // Assert expect(w1._id).toBeTruthy(); expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); - expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h._id.toString()); expect(w1.path).toStrictEqual('/'); expect(w1.domain.name).toStrictEqual(domainName); }); @@ -102,13 +111,13 @@ describe('Website Service', () => { const path = '/example/asdf/'; const c = await project(); const h = await host('1.1.1.1', c._id.toString()); - const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const p = await port(portNumber, h._id.toString(), c._id.toString()); const d = await domain(domainName, c._id.toString()); // Act const w1 = await websiteService.addWebsite( c._id.toString(), - h[0].ip, + h.ip, portNumber, domainName, path, @@ -117,7 +126,7 @@ describe('Website Service', () => { // Assert expect(w1._id).toBeTruthy(); expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); - expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h._id.toString()); expect(w1.path).toStrictEqual(path); expect(w1.domain.name).toStrictEqual(domainName); }); @@ -128,12 +137,12 @@ describe('Website Service', () => { const path = '/example/asdf/'; const c = await project(); const h = await host('1.1.1.1', c._id.toString()); - const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const p = await port(portNumber, h._id.toString(), c._id.toString()); // Act const w1 = await websiteService.addWebsite( c._id.toString(), - h[0].ip, + h.ip, portNumber, '', path, @@ -142,7 +151,7 @@ describe('Website Service', () => { // Assert expect(w1._id).toBeTruthy(); expect(w1.port.id.toString()).toStrictEqual(p._id.toString()); - expect(w1.host.id.toString()).toStrictEqual(h[0]._id.toString()); + expect(w1.host.id.toString()).toStrictEqual(h._id.toString()); expect(w1.path).toStrictEqual(path); expect(w1.domain).toBeNull(); }); @@ -154,11 +163,11 @@ describe('Website Service', () => { const domainName = 'example.com'; const c = await project(); const h = await host('1.1.1.1', c._id.toString()); - const p = await port(portNumber, h[0]._id.toString(), c._id.toString()); + const p = await port(portNumber, h._id.toString(), c._id.toString()); const d = await domain(domainName, c._id.toString()); const w1 = await websiteService.addWebsite( c._id.toString(), - h[0].ip, + h.ip, portNumber, domainName, path, @@ -167,7 +176,7 @@ describe('Website Service', () => { // Act const w2 = await websiteService.addWebsite( c._id.toString(), - h[0].ip, + h.ip, portNumber, domainName, path, @@ -180,13 +189,200 @@ describe('Website Service', () => { }); }); + describe('Get websites with filters', () => { + const ips = ['1.1.1.1', '1.1.1.1', '2.2.2.2', '2.2.2.2', '3.3.3.3']; + const ports = [80, 443, 80, 80, 8080]; + const domains = [ + 'example.com', + 'example.com', + 'example.com', + 'www.example.com', + '', + ]; + + it('Should filter a website by domain', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + const filter: WebsiteFilterModel = { domains: ['www.example.com'] }; + + // Act + const filteredWebsites = await websiteService.getAll( + 0, + ips.length, + filter, + ); + + // Assert + expect(filteredWebsites.length).toStrictEqual(1); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites[3]._id.toString(), + ); + }); + + it('Should filter websites by tag', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + const tag = await tagsService.create('website-test-tag', '#c0ffee'); + await websiteService.tagWebsite( + websites[2]._id.toString(), + tag._id.toString(), + true, + ); + const filter: WebsiteFilterModel = { tags: [tag._id.toString()] }; + + // Act + const filteredWebsites = await websiteService.getAll( + 0, + ips.length, + filter, + ); + + // Assert + expect(filteredWebsites.length).toStrictEqual(1); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites[2]._id.toString(), + ); + }); + + it('Should filter websites by host', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + const filter: WebsiteFilterModel = { hosts: ['1.1.1.1'] }; + + // Act + const filteredWebsites = await websiteService.getAll( + 0, + ips.length, + filter, + ); + + // Assert + expect(filteredWebsites.length).toStrictEqual(2); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites[0]._id.toString(), + ); + expect(filteredWebsites[1]._id.toString()).toStrictEqual( + websites[1]._id.toString(), + ); + }); + + it('Should filter websites by port', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + const filter: WebsiteFilterModel = { ports: [8080] }; + + // Act + const filteredWebsites = await websiteService.getAll( + 0, + ips.length, + filter, + ); + + // Assert + expect(filteredWebsites.length).toStrictEqual(1); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites[4]._id.toString(), + ); + }); + + it('Should filter websites by project', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + const websites2 = await bulkWebsites(['1.2.3.4'], [80], ['example.org']); + const filter: WebsiteFilterModel = { + project: [websites2[0].projectId.toString()], + }; + + // Act + const filteredWebsites = await websiteService.getAll( + 0, + ips.length, + filter, + ); + + // Assert + expect(filteredWebsites.length).toStrictEqual(1); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites2[0]._id.toString(), + ); + }); + + it('Should filter websites by merged', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + await websiteModel.updateOne( + { _id: { $eq: websites[1]._id } }, + { $set: { mergedInId: websites[2]._id } }, + ); + const filter: WebsiteFilterModel = { merged: true }; + + // Act + const filteredWebsites = await websiteService.getAll( + 0, + ips.length, + filter, + ); + + // Assert + expect(filteredWebsites.length).toStrictEqual(1); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites[1]._id.toString(), + ); + }); + + it('Should filter websites by blocked', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + await websiteService.batchEdit({ + block: true, + websiteIds: [websites[2]._id], + }); + const filter: WebsiteFilterModel = { blocked: true }; + + // Act + const filteredWebsites = await websiteService.getAll( + 0, + ips.length, + filter, + ); + + // Assert + expect(filteredWebsites.length).toStrictEqual(1); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites[2]._id.toString(), + ); + }); + + it('Should get websites with paging', async () => { + // Arrange + const websites = await bulkWebsites(ips, ports, domains); + await websiteService.batchEdit({ + block: true, + websiteIds: [websites[2]._id], + }); + const filter: WebsiteFilterModel = {}; + + // Act + const filteredWebsites = await websiteService.getAll(1, 2, filter); + + // Assert + expect(filteredWebsites.length).toStrictEqual(2); + expect(filteredWebsites[0]._id.toString()).toStrictEqual( + websites[2]._id.toString(), + ); + expect(filteredWebsites[1]._id.toString()).toStrictEqual( + websites[3]._id.toString(), + ); + }); + }); + async function project(name: string = '') { const ccDto: CreateProjectDto = { name: `${getName(testPrefix)}` }; return await projectService.addProject(ccDto); } - async function host(ip: string, projectId: string) { - return await hostService.addHosts([ip], projectId); + async function host(ip: string, projectId: string): Promise { + return await hostService.addHost(ip, projectId); } async function domain(name: string, projectId: string) { @@ -201,4 +397,57 @@ describe('Website Service', () => { ) { return await portService.addPort(hostId, projectId, port, protocol); } + + async function bulkWebsites( + ips: string[], + portNumbers: number[], + domainNames: string[], + paths: string[] = undefined, + ): Promise { + if ( + ips.length !== portNumbers.length || + ips.length !== domainNames.length || + (paths !== undefined && ips.length !== paths.length) + ) + throw new Error( + 'When creating websites in bulk, the arrays should have the same length', + ); + + const c = await project(); + const hosts: HostDocument[] = []; + for (const ip of ips) { + hosts.push(await host(ip, c._id.toString())); + } + + const ports: PortDocument[] = []; + for (let i = 0; i < portNumbers.length; ++i) { + ports.push( + await port(portNumbers[i], hosts[i]._id.toString(), c._id.toString()), + ); + } + + const domains: DomainDocument[] = []; + for (const d of domainNames) { + domains.push(await domain(d, c._id.toString())); + } + + if (paths === undefined) { + paths = Array(ips.length).fill('/'); + } + + const websites: WebsiteDocument[] = []; + for (let i = 0; i < hosts.length; ++i) { + websites.push( + await websiteService.addWebsite( + c._id.toString(), + hosts[i].ip, + ports[i].port, + domains[i].name, + paths[i], + ), + ); + } + + return websites; + } }); diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts index 1c54c23e7..9007a5cf9 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts @@ -1,8 +1,9 @@ -import { Injectable, Logger, NotImplementedException } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; -import { DeleteResult } from 'mongodb'; +import { DeleteResult, UpdateResult } from 'mongodb'; import { FilterQuery, Model, Types } from 'mongoose'; import { HttpNotFoundException } from '../../../../exceptions/http.exceptions'; +import escapeStringRegexp from '../../../../utils/escape-string-regexp'; import { WebsiteFinding } from '../../../findings/findings.service'; import { FindingsQueue } from '../../../job-queue/findings-queue'; import { TagsService } from '../../tags/tag.service'; @@ -12,6 +13,7 @@ import { DomainSummary } from '../domain/domain.summary'; import { Host } from '../host/host.model'; import { Port } from '../port/port.model'; import { WebsiteFilterModel } from './website-filter.model'; +import { BatchEditWebsitesDto } from './website.dto'; import { Website, WebsiteDocument } from './website.model'; @Injectable() @@ -33,6 +35,7 @@ export class WebsiteService { port: number, domain: string = undefined, path: string = '/', + ssl: boolean = undefined, ) { const projectIdObj = new Types.ObjectId(projectId); const existingPort = await this.portModel.findOne({ @@ -88,7 +91,7 @@ export class WebsiteService { return this.websiteModel.findOneAndUpdate( searchQuery, { - $set: { lastSeen: Date.now() }, + $set: { lastSeen: Date.now(), ssl: ssl ?? null }, $setOnInsert: { host: existingPort.host, domain: existingDomainSummary ?? null, @@ -125,6 +128,7 @@ export class WebsiteService { ip: string, port: number, path: string = '/', + ssl: boolean = false, ) { const host = await this.hostModel.findOne({ ip: { $eq: ip }, @@ -145,6 +149,7 @@ export class WebsiteService { path: path, port: port, fields: [], + ssl: ssl, }; // We will create a website finding @@ -182,6 +187,12 @@ export class WebsiteService { }); } + public async deleteMany(websiteIds: string[]): Promise { + return await this.websiteModel.deleteMany({ + _id: { $in: websiteIds.map((wid) => new Types.ObjectId(wid)) }, + }); + } + public async get(websiteId: string) { return await this.websiteModel.findById(websiteId); } @@ -193,7 +204,7 @@ export class WebsiteService { ): Promise { let query; if (filter) { - query = this.websiteModel.find(this.buildFilters(filter)); + query = this.websiteModel.find(await this.buildFilters(filter)); } else { query = this.websiteModel.find({}); } @@ -204,8 +215,126 @@ export class WebsiteService { return await query; } + public async batchEdit(dto: BatchEditWebsitesDto) { + const update: Partial = {}; + if (dto.block || dto.block === false) update.blocked = dto.block; + if (dto.block) update.blockedAt = Date.now(); + + return await this.websiteModel.updateMany( + { _id: { $in: dto.websiteIds.map((v) => new Types.ObjectId(v)) } }, + update, + ); + } + private async buildFilters(filter: WebsiteFilterModel) { - throw new NotImplementedException(); + const finalFilter = {}; + + // Filter by host ip + if (filter.hosts) { + const hostsRegex = filter.hosts + .filter((x) => x) + .map((x) => x.toLowerCase().trim()) + .map((x) => escapeStringRegexp(x)) + .map((x) => new RegExp(`.*${x}.*`)); + + if (hostsRegex.length > 0) { + const hosts = await this.hostModel.find( + { ip: { $in: hostsRegex } }, + '_id', + ); + if (hosts) finalFilter['host.id'] = { $in: hosts.map((h) => h._id) }; + } + } + + // Filter by domain + if (filter.domains) { + const domainsRegex = filter.domains + .filter((x) => x) + .map((x) => x.toLowerCase().trim()) + .map((x) => escapeStringRegexp(x)) + .map((x) => new RegExp(`.*${x}.*`)); + + if (domainsRegex.length > 0) { + const domains = await this.domainModel.find( + { name: { $in: domainsRegex } }, + '_id', + ); + if (domains) + finalFilter['domain.id'] = { $in: domains.map((d) => d._id) }; + } + } + + // Filter by port + if (filter.ports) { + const ports = await this.portModel.find( + { port: { $in: filter.ports } }, + '_id', + ); + if (ports) finalFilter['port.id'] = { $in: ports.map((p) => p._id) }; + } + + // Filter by project + if (filter.project) { + const projectIds = filter.project + .filter((x) => x) + .map((x) => new Types.ObjectId(x)); + + if (projectIds.length > 0) { + finalFilter['projectId'] = { $in: projectIds }; + } + } + + // Filter by tag + if (filter.tags) { + const preppedTagsArray = filter.tags + .filter((x) => x) + .map((x) => x.toLowerCase()) + .map((x) => new Types.ObjectId(x)); + + if (preppedTagsArray.length > 0) { + finalFilter['tags'] = { + $all: preppedTagsArray.map((t) => new Types.ObjectId(t)), + }; + } + } + + // Filter by createdAt + if (filter.firstSeenStartDate || filter.firstSeenEndDate) { + let createdAtFilter = {}; + + if (filter.firstSeenStartDate && filter.firstSeenEndDate) { + createdAtFilter = [ + { createdAt: { $gte: filter.firstSeenStartDate } }, + { createdAt: { $lte: filter.firstSeenEndDate } }, + ]; + finalFilter['$and'] = createdAtFilter; + } else { + if (filter.firstSeenStartDate) + createdAtFilter = { $gte: filter.firstSeenStartDate }; + else if (filter.firstSeenEndDate) + createdAtFilter = { $lte: filter.firstSeenEndDate }; + finalFilter['createdAt'] = createdAtFilter; + } + } + + // Filter by blocked + if (filter.blocked === false) { + finalFilter['$or'] = [ + { blocked: { $exists: false } }, + { blocked: { $eq: false } }, + ]; + } else if (filter.blocked === true) { + finalFilter['blocked'] = { $eq: true }; + } + + // Filter by merged + if (filter.merged === false) { + finalFilter['mergedInId'] = { $exists: false }; + } else if (filter.merged === true) { + finalFilter['mergedInId'] = { $exists: true }; + } + + return finalFilter; } public async keyIsBlocked(correlationKey: string) { @@ -216,6 +345,40 @@ export class WebsiteService { return website && (!!website.mergedInId || website.blocked); } + public async count(filter: WebsiteFilterModel = null) { + if (!filter) { + return await this.websiteModel.estimatedDocumentCount(); + } else { + return await this.websiteModel.countDocuments( + await this.buildFilters(filter), + ); + } + } + + public async tagWebsite( + websiteId: string, + tagId: string, + isTagged: boolean, + ): Promise { + const website = await this.websiteModel.findById(websiteId); + if (!website) throw new HttpNotFoundException(); + + if (!isTagged) { + return await this.websiteModel.updateOne( + { _id: { $eq: new Types.ObjectId(websiteId) } }, + { $pull: { tags: new Types.ObjectId(tagId) } }, + ); + } else { + if (!(await this.tagsService.tagExists(tagId))) + throw new HttpNotFoundException(); + + return await this.websiteModel.updateOne( + { _id: { $eq: new Types.ObjectId(websiteId) } }, + { $addToSet: { tags: new Types.ObjectId(tagId) } }, + ); + } + } + /** * TODO: * diff --git a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts index 490e6bda8..df8e0ff1e 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts @@ -55,6 +55,7 @@ export class CustomFindingHandler extends JobFindingHandlerBase { } protected async executeCore(command: WebsiteCommand) { + console.log(command.finding); await this.websiteService.addWebsite( command.projectId, command.finding.ip, command.finding.port, command.finding.domain, command.finding.path, + command.finding.ssl || command.finding.ssl === false + ? command.finding.ssl + : undefined, ); } } diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts index ac51125f4..639f0bcf8 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts @@ -59,6 +59,7 @@ export class WebsiteFinding extends FindingBase { port: number; domain: string = ''; path: string = '/'; + ssl?: boolean; } export class CreateCustomFinding extends FindingBase { diff --git a/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts b/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts index e1cd0ed31..81cc37105 100644 --- a/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts +++ b/packages/backend/jobs-manager/service/src/modules/job-queue/kafka-findings-queue.ts @@ -30,6 +30,7 @@ export class KafkaFindingsQueue implements FindingsQueue { value: JSON.stringify({ FindingsJson: JSON.stringify({ findings: findings }), JobId: jobId, + Timestamp: Date.now(), }), }, ]; diff --git a/packages/frontend/stalker-app/src/app/api/websites/websites.service.ts b/packages/frontend/stalker-app/src/app/api/websites/websites.service.ts new file mode 100644 index 000000000..a703b9500 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/api/websites/websites.service.ts @@ -0,0 +1,98 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { DateRange } from '@angular/material/datepicker'; +import { Observable, firstValueFrom, map } from 'rxjs'; +import { environment } from 'src/environments/environment'; +import { Page } from '../../shared/types/page.type'; +import { Website } from '../../shared/types/websites/website.type'; +import { filtersToParams } from '../../utils/filters-to-params'; + +@Injectable({ + providedIn: 'root', +}) +export class WebsitesService { + constructor(private http: HttpClient) {} + + public getPage( + page: number, + pageSize: number, + filters: any = undefined, + firstSeenDateRange: DateRange = new DateRange(null, null) + ): Observable> { + let params = filtersToParams(filters); + params = params.append('page', page); + params = params.append('pageSize', pageSize); + + if (firstSeenDateRange.start) params = params.append('firstSeenStartDate', firstSeenDateRange.start.getTime()); + if (firstSeenDateRange.end) params = params.append('firstSeenEndDate', firstSeenDateRange.end.getTime()); + + return this.http.get>>(`${environment.fmUrl}/websites`, { params }).pipe( + map((page: Page>) => { + const websitePage: Page = { + totalRecords: page.totalRecords, + items: [], + }; + + for (const website of page.items) { + websitePage.items.push({ + url: this.buildUrl( + website.domain ? website.domain.name : '', + website.host.ip, + website.port.port, + website.path, + website.ssl + ), + ...website, + }); + } + + return websitePage; + }) + ); + } + + public async tagWebsite(websiteId: string, tagId: string, isTagged: boolean) { + return await firstValueFrom( + this.http.put(`${environment.fmUrl}/websites/${websiteId}/tags`, { tagId: tagId, isTagged: isTagged }) + ); + } + + public getWebsite(websiteId: string): Observable { + return (>>this.http.get(`${environment.fmUrl}/websites/${websiteId}`)).pipe( + map((website) => { + return { + url: this.buildUrl( + website.domain ? website.domain.name : '', + website.host.ip, + website.port.port, + website.path, + website.ssl + ), + ...website, + }; + }) + ); + } + + public async delete(websiteId: string) { + return await firstValueFrom(this.http.delete(`${environment.fmUrl}/websites/${websiteId}`)); + } + + public async deleteMany(websiteIds: string[]) { + return await firstValueFrom( + this.http.delete(`${environment.fmUrl}/websites/`, { body: { websiteIds: websiteIds } }) + ); + } + + public async block(websiteIds: string[], block: boolean) { + return await firstValueFrom(this.http.patch(`${environment.fmUrl}/websites/`, { websiteIds: websiteIds, block })); + } + + public buildUrl(domain: string, ip: string, port: number, path: string, ssl: boolean): string { + let url = ssl ? 'https://' : 'http://'; + url += domain ? domain : ip; + url += port === 80 || port === 443 ? '' : ':' + port.toString(); + url += path; + return url; + } +} diff --git a/packages/frontend/stalker-app/src/app/app-routing.module.ts b/packages/frontend/stalker-app/src/app/app-routing.module.ts index 4f28c42d6..d6423edfd 100644 --- a/packages/frontend/stalker-app/src/app/app-routing.module.ts +++ b/packages/frontend/stalker-app/src/app/app-routing.module.ts @@ -49,6 +49,10 @@ const routes: Routes = [ path: 'ports', loadChildren: () => import('./modules/ports/ports.module').then((m) => m.PortsListModule), }, + { + path: 'websites', + loadChildren: () => import('./modules/websites/websites.module').then((m) => m.WebsitesListModule), + }, { path: 'tags', loadChildren: () => import('./modules/tags/tags.module').then((m) => m.TagsModule), diff --git a/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.html b/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.html new file mode 100644 index 000000000..c9f50154d --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.html @@ -0,0 +1,122 @@ + + +
+ Websites + +
+ + + +
+
+ + + + @if (((dataSource$ | async) && (projects$ | async) && (tags$ | async)) || true) { + + + + Url + + {{ element.url }} + @if (element.blocked) { + + } + + + + + + Domain + + @if (element.domain) { + {{ + element.domain.name + }} + } + + + + + + Host + + {{ element.host.ip }} + + + + + + Port + +
+ {{ element.port.port }} +
+ + + + + + Path + + {{ element.path }} + + + + + + Project + + + + + + + + Tags + +
+ @for (tag of element.tags; track tag) { + {{ + (tags | whereId: [tag])?.text + }} + } + +
+ + + + + } + + diff --git a/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.scss b/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.scss new file mode 100644 index 000000000..fc0ac9a86 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.scss @@ -0,0 +1,16 @@ +mat-card { + overflow: hidden; +} + +mat-dialog-content { + display: flex; + flex-direction: column; +} + +mat-dialog-actions { + justify-content: center; +} + +app-blocked-pill-tag { + margin-left: 5px; +} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.ts b/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.ts new file mode 100644 index 000000000..45a6089a0 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/list-websites/list-websites.component.ts @@ -0,0 +1,286 @@ +import { SelectionModel } from '@angular/cdk/collections'; +import { BreakpointObserver, BreakpointState, Breakpoints } from '@angular/cdk/layout'; +import { CommonModule } from '@angular/common'; +import { Component } from '@angular/core'; +import { FormsModule, ReactiveFormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { MatCardModule } from '@angular/material/card'; +import { DateRange } from '@angular/material/datepicker'; +import { MatDialog, MatDialogModule } from '@angular/material/dialog'; +import { MatFormFieldModule } from '@angular/material/form-field'; +import { MatIconModule } from '@angular/material/icon'; +import { MatInputModule } from '@angular/material/input'; +import { PageEvent } from '@angular/material/paginator'; +import { MatSelectModule } from '@angular/material/select'; +import { MatTableDataSource, MatTableModule } from '@angular/material/table'; +import { Title } from '@angular/platform-browser'; +import { RouterModule } from '@angular/router'; +import { ToastrService } from 'ngx-toastr'; +import { BehaviorSubject, map, switchMap, tap } from 'rxjs'; +import { ProjectsService } from 'src/app/api/projects/projects.service'; +import { TagsService } from 'src/app/api/tags/tags.service'; +import { ProjectCellComponent } from 'src/app/shared/components/project-cell/project-cell.component'; +import { Page } from 'src/app/shared/types/page.type'; +import { ProjectSummary } from 'src/app/shared/types/project/project.summary'; +import { Tag } from 'src/app/shared/types/tag.type'; +import { + ElementMenuItems, + FilteredPaginatedTableComponent, +} from 'src/app/shared/widget/filtered-paginated-table/filtered-paginated-table.component'; +import { BlockedPillTagComponent } from 'src/app/shared/widget/pill-tag/blocked-pill-tag.component'; +import { WebsitesService } from '../../../api/websites/websites.service'; +import { SharedModule } from '../../../shared/shared.module'; +import { Website } from '../../../shared/types/websites/website.type'; +import { defaultNewTimeMs } from '../../../shared/widget/pill-tag/new-pill-tag.component'; +import { WebsiteInteractionsService } from '../websites-interactions.service'; + +@Component({ + standalone: true, + imports: [ + CommonModule, + MatCardModule, + MatIconModule, + MatDialogModule, + MatFormFieldModule, + MatSelectModule, + SharedModule, + FormsModule, + MatButtonModule, + MatTableModule, + ReactiveFormsModule, + MatInputModule, + ProjectCellComponent, + FilteredPaginatedTableComponent, + RouterModule, + BlockedPillTagComponent, + ], + selector: 'app-list-websites', + templateUrl: './list-websites.component.html', + styleUrls: ['./list-websites.component.scss'], +}) +export class ListWebsitesComponent { + dataLoading = true; + displayedColumns: string[] = ['select', 'url', 'domain', 'port', 'ip', 'project', 'tags', 'menu']; + filterOptions: string[] = ['domain', 'host', 'port', 'project', 'tags', 'is']; + public readonly noDataMessage = $localize`:No website found|No website was found:No website found`; + + dataSource = new MatTableDataSource(); + currentPage: PageEvent = this.generateFirstPageEvent(); + currentFilters: string[] = ['-is: blocked', '-is: merged']; + currentPage$ = new BehaviorSubject(this.currentPage); + count = 0; + selection = new SelectionModel(true, []); + currentDateRange: DateRange = new DateRange(null, null); + startDate: Date | null = null; + + dataSource$ = this.currentPage$.pipe( + tap((currentPage) => { + this.currentPage = currentPage; + }), + switchMap((currentPage) => { + const filters = this.buildFilters(this.currentFilters); + return this.websitesService.getPage(currentPage.pageIndex, currentPage.pageSize, filters, this.currentDateRange); + }), + map((data: Page) => { + this.dataSource = new MatTableDataSource(data.items); + this.count = data.totalRecords; + this.dataLoading = false; + return data; + }) + ); + + projects: ProjectSummary[] = []; + projects$ = this.projectsService.getAllSummaries().pipe(tap((x) => (this.projects = x))); + + tags: Tag[] = []; + tags$ = this.tagsService.getTags().pipe( + map((next: any[]) => { + const tagsArr: Tag[] = []; + for (const tag of next) { + tagsArr.push({ _id: tag._id, text: tag.text, color: tag.color }); + } + this.tags = tagsArr; + return this.tags; + }) + ); + + private generateFirstPageEvent() { + const p = new PageEvent(); + p.pageIndex = 0; + p.pageSize = 10; + this.currentPage = p; + return p; + } + + private screenSize$ = this.bpObserver.observe([ + Breakpoints.XSmall, + Breakpoints.Small, + Breakpoints.Large, + Breakpoints.XLarge, + ]); + + public displayColumns$ = this.screenSize$.pipe( + map((screen: BreakpointState) => { + if (screen.breakpoints[Breakpoints.XSmall]) return ['select', 'website', 'project', 'menu']; + else if (screen.breakpoints[Breakpoints.Small]) + return ['select', 'website', 'domain', 'port', 'ip', 'project', 'tags', 'menu']; + else if (screen.breakpoints[Breakpoints.Medium]) + return ['select', 'website', 'domain', 'port', 'ip', 'project', 'tags', 'menu']; + return this.displayedColumns; + }) + ); + + pageChange(event: PageEvent) { + this.dataLoading = true; + this.currentPage$.next(event); + } + + constructor( + private bpObserver: BreakpointObserver, + private projectsService: ProjectsService, + private websitesService: WebsitesService, + private websitesInteractor: WebsiteInteractionsService, + private toastr: ToastrService, + private tagsService: TagsService, + public dialog: MatDialog, + private titleService: Title + ) { + this.titleService.setTitle($localize`:Websites list page title|:Websites`); + } + + filtersChange(filters: string[]) { + this.currentFilters = filters; + this.dataLoading = true; + this.currentPage$.next(this.currentPage); + } + + dateRangeFilterChange(range: DateRange) { + this.currentDateRange = range; + this.dataLoading = true; + this.currentPage$.next(this.currentPage); + } + + buildFilters(stringFilters: string[]): any { + const SEPARATOR = ':'; + const NEGATING_CHAR = '-'; + const filterObject: any = {}; + const tags = []; + const ports = []; + const hosts = []; + const domains = []; + const projects = []; + let blocked: boolean | null = null; + let merged: boolean | null = null; + + for (const filter of stringFilters) { + if (filter.indexOf(SEPARATOR) === -1) continue; + + const keyValuePair = filter.split(SEPARATOR); + + if (keyValuePair.length !== 2) continue; + + let key = keyValuePair[0].trim().toLowerCase(); + const value = keyValuePair[1].trim().toLowerCase(); + const negated = key.length > 0 && key[0] === NEGATING_CHAR; + if (negated) key = key.substring(1); + + if (!key || !value) continue; + + switch (key) { + case 'project': + const project = this.projects.find((c) => c.name.trim().toLowerCase() === value.trim().toLowerCase()); + if (project) projects.push(project.id); + else + this.toastr.warning( + $localize`:Project does not exist|The given project name is not known to the application:Project name not recognized` + ); + break; + case 'host': + if (value) hosts.push(value.trim().toLowerCase()); + break; + case 'domain': + if (value) domains.push(value.trim().toLowerCase()); + break; + case 'tags': + const tag = this.tags.find((t) => t.text.trim().toLowerCase() === value.trim().toLowerCase()); + if (tag) tags.push(tag._id); + else + this.toastr.warning( + $localize`:Tag does not exist|The given tag is not known to the application:Tag not recognized` + ); + break; + case 'port': + ports.push(value); + break; + case 'is': + switch (value) { + case 'blocked': + blocked = !negated; + break; + case 'merged': + merged = !negated; + break; + } + break; + } + } + if (tags?.length) filterObject['tags'] = tags; + if (ports?.length) filterObject['ports'] = ports; + if (hosts?.length) filterObject['hosts'] = hosts; + if (domains?.length) filterObject['domains'] = domains; + if (projects?.length) filterObject['project'] = projects; + if (blocked !== null) filterObject['blocked'] = blocked; + if (merged !== null) filterObject['merged'] = merged; + return filterObject; + } + + dateFilter(event: MouseEvent) { + event.stopPropagation(); + this.startDate = new Date(Date.now() - defaultNewTimeMs); + } + + public async deleteBatch(domains: Website[]) { + const result = await this.websitesInteractor.deleteBatch(domains, this.projects); + if (result) { + this.selection.clear(); + this.currentPage$.next(this.currentPage); + } + } + + public async blockBatch(domains: Website[]) { + const result = await this.websitesInteractor.blockBatch(domains, this.projects); + if (result) { + this.selection.clear(); + this.currentPage$.next(this.currentPage); + } + } + + public async block(domainId: string, block: boolean) { + const result = await this.websitesInteractor.block(domainId, block); + if (result) { + this.selection.clear(); + this.currentPage$.next(this.currentPage); + } + } + + public generateMenuItem = (element: Website): ElementMenuItems[] => { + if (!element) return []; + const menuItems: ElementMenuItems[] = []; + + menuItems.push({ + action: () => this.block(element._id, !element.blocked), + icon: element.blocked ? 'thumb_up ' : 'block', + label: element.blocked + ? $localize`:Unblock website|Unblock website:Unblock` + : $localize`:Block website|Block website:Block`, + }); + + menuItems.push({ + action: () => this.deleteBatch([element]), + icon: 'delete', + label: $localize`:Delete website|Delete website:Delete`, + }); + + return menuItems; + }; +} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html new file mode 100644 index 000000000..f9312b304 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html @@ -0,0 +1,197 @@ +@if (website$ | async; as wsite) { +
+ + + @if (wsite.blocked) { + + } + +

+ @if (wsite.ssl) { + lock + } @else { + no_encryption + } +

+
+ +
+
+ +
+
+ @if (wsite.blocked) { + + Blocked + + {{ wsite.blockedAt | timeAgo }} + + + } + + Last seen + + {{ wsite.lastSeen | timeAgo }} + + + + First seen + + {{ wsite.createdAt | timeAgo }} + + + + @if ((tagsSelectItems$ | async) && mergedTags$ | async; as websiteTags) { + + Tags + + — + + + + @if (website$ | async) { + @if (!websiteTags.length && !newPillTag.isNew()) { + No tags yet + + } +
+ @for (tag of websiteTags; track tag) { + {{ + (tags | whereId: [tag])?.text + }} + } + +
+ } + + } + + + Website + {{ wsite.url }} + + + + Domain + @if (wsite.domain) { + {{ wsite.domain.name }} + } @else { + No domain yet + } + + + + IP address + {{ wsite.host.ip }} + @for (altHost of wsite.alternativeHosts; track altHost) { + {{ altHost.ip }} + } + + + + Port + + + + + + Path + +
+ {{ wsite.path }} +
+ + + + + + Merged domains + + @for (altDomain of wsite.alternativeDomains; track altDomain) { + {{ altDomain.name }} + } + @if (!wsite.alternativeDomains || !wsite.alternativeDomains.length) { + No domains yet + } + + + + Merged hosts + @if (hostPorts$ | async; as allHostPorts) { + @if (allHostPorts.length > 5 && !showAllHosts ? allHostPorts.slice(0, 5) : allHostPorts; as hostPorts) { + @for (hostPort of hostPorts; track hostPort) { +
+ {{ hostPort.hostSummary.ip }} + @if ( + showAllPorts[hostPort.hostSummary.ip] ? hostPort.websitePorts : hostPort.websitePorts.slice(0, 5); + as ports + ) { +
    + @for (port of ports; track port; let last = $last) { +
  • + {{ port.port }} +
    • + } +
+ } + @if (hostPort.websitePorts.length > 5) { +
+ +
+ } +
+ } + } + @if (allHostPorts.length > 5) { + + } + } @else { + No data yet + } + + + + + + +
+ + +
+ + +
+
+} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss new file mode 100644 index 000000000..1ad22474c --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss @@ -0,0 +1,89 @@ +:host { + position: absolute; + top: 0; + bottom: 0; + right: 0; + left: 0; + display: flex; + flex-direction: column; + align-items: stretch; + padding: 16px; + padding-top: 0; +} + +.content-wrapper { + box-sizing: border-box; + display: grid; + grid-template-columns: 1fr 200px; + max-width: 1100px; + width: 100%; + gap: var(--normal-gap-size); + align-self: center; +} + +.show-all-button-wrapper { + display: flex; + justify-content: center; + + .show-all-button { + margin-top: 8px; + opacity: 0.6; + } +} + +.context { + display: flex; + flex-direction: column; + gap: var(--normal-gap-size); +} + +panel-section-title { + width: 100%; + display: flex; + .manage-tags-menu { + font-size: 0.9em; + font-style: italic; + display: inline-flex; + width: 100%; + app-text-select-menu { + margin-left: 3px; + width: 100%; + } + } +} + +.context { + display: flex; + flex-direction: column; + gap: var(--normal-gap-size); +} + +.host { + margin-bottom: 4px; + + ul { + padding-inline-start: 16px; + margin-top: 2px; + margin-bottom: 0px; + list-style-type: ''; + + a { + cursor: pointer; + font-weight: 400; + margin-left: -4px; + } + } + + .show-more { + margin-left: 14px; + } +} + +.secure-icon { + padding-top: 16px; + padding-bottom: 8px; + display: flex; + align-items: center; + font-weight: 500; + font-size: 18px; +} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts new file mode 100644 index 000000000..556a376fe --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts @@ -0,0 +1,292 @@ +import { CommonModule } from '@angular/common'; +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, ElementRef, OnDestroy, ViewChild } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { MatCardModule } from '@angular/material/card'; +import { MatDialog } from '@angular/material/dialog'; +import { MatFormFieldModule } from '@angular/material/form-field'; +import { MatIconModule } from '@angular/material/icon'; +import { MatInputModule } from '@angular/material/input'; +import { MatListModule } from '@angular/material/list'; +import { MatMenuModule } from '@angular/material/menu'; +import { MatPaginatorModule } from '@angular/material/paginator'; +import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; +import { MatSidenavModule } from '@angular/material/sidenav'; +import { MatTableModule } from '@angular/material/table'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { Title } from '@angular/platform-browser'; +import { ActivatedRoute, Router, RouterModule } from '@angular/router'; +import { ToastrService } from 'ngx-toastr'; +import { + BehaviorSubject, + Observable, + Subject, + combineLatest, + firstValueFrom, + forkJoin, + map, + merge, + shareReplay, + switchMap, + tap, +} from 'rxjs'; +import { ProjectsService } from 'src/app/api/projects/projects.service'; +import { TagsService } from 'src/app/api/tags/tags.service'; +import { Domain } from 'src/app/shared/types/domain/domain.interface'; +import { DomainSummary } from 'src/app/shared/types/domain/domain.summary'; +import { ProjectSummary } from 'src/app/shared/types/project/project.summary'; +import { Tag } from 'src/app/shared/types/tag.type'; +import { BlockedPillTagComponent } from 'src/app/shared/widget/pill-tag/blocked-pill-tag.component'; +import { TextMenuComponent } from 'src/app/shared/widget/text-menu/text-menu.component'; +import { PortsService } from '../../../api/ports/ports.service'; +import { WebsitesService } from '../../../api/websites/websites.service'; +import { AppHeaderComponent } from '../../../shared/components/page-header/page-header.component'; +import { PanelSectionModule } from '../../../shared/components/panel-section/panel-section.module'; +import { SharedModule } from '../../../shared/shared.module'; +import { HostSummary } from '../../../shared/types/host/host.summary'; +import { Port } from '../../../shared/types/ports/port.interface'; +import { Website } from '../../../shared/types/websites/website.type'; +import { NewPillTagComponent } from '../../../shared/widget/pill-tag/new-pill-tag.component'; +import { SelectItem } from '../../../shared/widget/text-select-menu/text-select-menu.component'; +import { FindingsModule } from '../../findings/findings.module'; +import { WebsiteInteractionsService } from '../websites-interactions.service'; + +@Component({ + standalone: true, + imports: [ + CommonModule, + SharedModule, + MatCardModule, + MatIconModule, + MatListModule, + MatFormFieldModule, + MatTableModule, + MatPaginatorModule, + MatSidenavModule, + MatButtonModule, + MatInputModule, + MatProgressSpinnerModule, + MatMenuModule, + FormsModule, + FindingsModule, + RouterModule, + PanelSectionModule, + AppHeaderComponent, + MatTooltipModule, + TextMenuComponent, + BlockedPillTagComponent, + ], + selector: 'app-view-website', + templateUrl: './view-website.component.html', + styleUrls: ['./view-website.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ViewWebsiteComponent implements OnDestroy { + public menuResizeObserver$?: ResizeObserver; + + @ViewChild('newPillTag', { read: ElementRef, static: false }) + newPillTag!: NewPillTagComponent; + + @ViewChild('managementPanelSection', { read: ElementRef, static: false }) + set managementPanelSection(managementPanelSection: ElementRef) { + if (managementPanelSection && !this._managementPanelSection) { + this._managementPanelSection = managementPanelSection; + this.menuResizeObserver$ = new ResizeObserver((resize) => {}); + this.menuResizeObserver$.observe(this._managementPanelSection.nativeElement); + } + } + + _managementPanelSection!: ElementRef; + displayedColumns: string[] = ['domainName']; + public manageTags: string = $localize`:Manage Tags|Manage Tags:Manage Tags`; + public filterTags: string = $localize`:Filter Tags|Filter Tags:Filter Tags`; + public emptyTags: string = $localize`:No Tags|List of tags is empty:No Tags Available`; + public manageWebsiteText: string = $localize`:Manage website|Manage the website element:Manage website`; + public secureConnectionTooltip: string = $localize`:Connection secure|:Connections to this website are encrypted`; + public insecureConnectionTooltip: string = $localize`:Connection insecure|:Connections to this website are not encrypted`; + + // Drawer + public currentDetailsId: string | null = null; + public selectedDomain: DomainSummary | null = null; + public domainDetails$: Observable | null = null; + public websiteDetails$: Observable | null = null; + public selectedItemCorrelationKey$ = new Subject(); + + projects: ProjectSummary[] = []; + projects$ = this.projectsService.getAllSummaries().pipe( + map((next: any[]) => { + const comp: ProjectSummary[] = []; + for (const project of next) { + comp.push({ id: project._id, name: project.name }); + } + this.projects = comp; + return this.projects; + }) + ); + + public showAllPorts: { [ip: string]: boolean } = {}; + public showAllHosts: boolean = false; + + public websiteId$ = this.route.params.pipe(map((params) => params['id'] as string)); + public website!: Website; + public websiteId!: string; + public website$ = this.websiteId$.pipe( + tap((id: string) => { + this.websiteId = id; + }), + switchMap((id: string) => { + return this.websitesService.getWebsite(id); + }), + tap((website: Website) => { + this.website = website; + }), + shareReplay(1) + ); + + public hostPorts$ = this.website$.pipe( + map((website) => { + const allPortSummaries = []; + if (website.alternativePorts) { + for (const portSummary of website.alternativePorts) { + allPortSummaries.push(portSummary); + } + } + + const portObservables: Observable[] = []; + for (const portSummary of allPortSummaries) { + portObservables.push(this.portsService.getPort(portSummary.id)); + } + + return forkJoin(portObservables); + }), + switchMap((port) => port), + map((ports: Port[]) => { + const allHostSummaries: HostSummary[] = []; + if (this.website.alternativeHosts) { + for (const hostSummary of this.website.alternativeHosts) { + allHostSummaries.push(hostSummary); + } + } + const hostPorts: { hostSummary: HostSummary; websitePorts: Port[] }[] = []; + + for (const hostSummary of allHostSummaries) { + hostPorts.push({ + hostSummary: hostSummary, + websitePorts: ports.filter((p) => p.host.id === hostSummary.id), + }); + } + + return hostPorts; + }) + ); + + public websiteTitle$ = this.website$.pipe( + tap((website) => this.titleService.setTitle($localize`:Website page title|:Website · ${website.url}`)), + shareReplay(1) + ); + + public port$ = this.website$.pipe( + map((website) => website.port), + shareReplay(1) + ); + + public websiteTagsCache: string[] = []; + public websiteTagsSubject$ = new BehaviorSubject([]); + public websiteTags$ = this.website$.pipe( + map((website) => { + this.websiteTagsCache = website.tags ? website.tags : []; + return this.websiteTagsCache; + }) + ); + + tags: (Tag & SelectItem)[] = []; + allTags$ = this.tagsService.getTags().pipe( + map((next: any[]) => { + const tagsArr: Tag[] = []; + for (const tag of next) { + tagsArr.push({ _id: tag._id, text: tag.text, color: tag.color }); + } + return tagsArr; + }) + ); + + public tagsSelectItems$ = combineLatest([this.websiteTags$, this.allTags$]).pipe( + map(([websiteTags, allTags]) => { + const tagsArr: (Tag & SelectItem)[] = []; + for (const tag of allTags) { + if (websiteTags?.includes(tag._id)) { + tagsArr.push({ _id: tag._id, text: tag.text, color: tag.color, isSelected: true }); + } else { + tagsArr.push({ _id: tag._id, text: tag.text, color: tag.color, isSelected: false }); + } + } + this.tags = tagsArr; + return tagsArr; + }) + ); + + public mergedTags$ = merge(this.websiteTagsSubject$, this.websiteTags$); + + /** + * + * @param item A SelectItem, but contains all the attributes of a Tag. + */ + public async itemSelected(item: SelectItem) { + try { + const tagId = item['_id']; + if (!this.websiteId) return; + const tagIndex = this.websiteTagsCache.findIndex((tag: string) => tag === tagId); + + if (tagIndex === -1 && item.color !== undefined) { + // Tag not found, adding it + await this.websitesService.tagWebsite(this.websiteId, tagId, true); + this.websiteTagsCache.push(tagId); + } else { + // Tag was found, removing it + await this.websitesService.tagWebsite(this.websiteId, tagId, false); + this.websiteTagsCache.splice(tagIndex, 1); + } + this.websiteTagsSubject$.next(this.websiteTagsCache); + } catch (err) { + this.toastr.error($localize`:Error while tagging|Error while tagging an item:Error while tagging`); + } + } + + ngOnDestroy(): void { + if (this.menuResizeObserver$) { + this.menuResizeObserver$.unobserve(this._managementPanelSection.nativeElement); + } + } + + public async deleteWebsite() { + const result = await this.websitesInteractor.deleteBatch([this.website], this.projects); + if (result) { + this.router.navigate([`/websites/`]); + } + } + + public async blockWebsite(block: boolean) { + const result = await this.websitesInteractor.block(this.websiteId, block); + if (result) { + const p = await firstValueFrom(this.websitesService.getWebsite(this.websiteId)); + this.website.blocked = p.blocked; + this.website.blockedAt = p.blockedAt; + this.cdr.markForCheck(); + this.dialog.closeAll(); + } + } + + constructor( + private route: ActivatedRoute, + private projectsService: ProjectsService, + private tagsService: TagsService, + private titleService: Title, + private websitesService: WebsitesService, + private websitesInteractor: WebsiteInteractionsService, + private portsService: PortsService, + private toastr: ToastrService, + private router: Router, + public dialog: MatDialog, + private cdr: ChangeDetectorRef + ) {} +} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/websites-interactions.service.ts b/packages/frontend/stalker-app/src/app/modules/websites/websites-interactions.service.ts new file mode 100644 index 000000000..851b825c9 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/websites-interactions.service.ts @@ -0,0 +1,161 @@ +import { Injectable } from '@angular/core'; +import { MatDialog } from '@angular/material/dialog'; +import { ToastrService } from 'ngx-toastr'; +import { firstValueFrom } from 'rxjs'; +import { ProjectSummary } from 'src/app/shared/types/project/project.summary'; +import { + ConfirmDialogComponent, + ConfirmDialogData, +} from 'src/app/shared/widget/confirm-dialog/confirm-dialog.component'; +import { WebsitesService } from '../../api/websites/websites.service'; +import { Website } from '../../shared/types/websites/website.type'; + +@Injectable({ providedIn: 'root' }) +export class WebsiteInteractionsService { + constructor( + private websitesService: WebsitesService, + private dialog: MatDialog, + private toastr: ToastrService + ) {} + + public deleteBatch(websites: Pick[], projects?: ProjectSummary[]) { + let data: ConfirmDialogData; + if (websites.length > 0) { + data = { + text: $localize`:Confirm delete websites|Confirmation message asking if the user really wants to delete the selected websites:Do you really wish to delete these websites permanently ?`, + title: $localize`:Deleting websites|Title of a page to delete selected websites:Deleting websites`, + primaryButtonText: $localize`:Cancel|Cancel current action:Cancel`, + dangerButtonText: $localize`:Delete permanently|Confirm that the user wants to delete the item permanently:Delete permanently`, + listElements: this.toBulletPoints(websites, projects), + onPrimaryButtonClick: () => { + this.dialog.closeAll(); + }, + onDangerButtonClick: async (close) => { + const ids = websites.map((d) => d._id); + await this.websitesService.deleteMany(ids); + this.toastr.success( + $localize`:Websites deleted|Confirm the successful deletion of a website:Websites deleted successfully` + ); + close(true); + }, + }; + } else { + data = { + text: $localize`:Select websites again|No websites were selected so there is nothing to delete:Select the websites to delete and try again.`, + title: $localize`:Nothing to delete|Tried to delete something, but there was nothing to delete:Nothing to delete`, + primaryButtonText: $localize`:Ok|Accept or confirm:Ok`, + onPrimaryButtonClick: (close) => { + close(false); + }, + }; + } + + return firstValueFrom( + this.dialog + .open(ConfirmDialogComponent, { + data, + restoreFocus: false, + }) + .afterClosed() + ); + } + + public async blockBatch(websites: Pick[], projects?: ProjectSummary[]) { + let data: ConfirmDialogData; + if (websites.length > 0) { + const block = async (close: (result: boolean) => void, block: boolean) => { + try { + await this.websitesService.block( + websites.map((s) => s._id), + block + ); + this.toastr.success( + block + ? $localize`:Websites blocked|Blocked a website:Websites successfully blocked` + : $localize`:Websites unblocked|Unblocked a website:Websites successfully unblocked` + ); + close(true); + } catch { + this.toastr.error($localize`:Error blocking|Error while blocking a website:Error blocking websites`); + close(false); + } + }; + + data = { + text: $localize`:Confirm block websites|Confirmation message asking if the user wants to block the selected websites:Do you wish to block or unblock these websites?`, + title: $localize`:Blocking websites|Title of a page to block selected websites:Blocking websites`, + primaryButtonText: $localize`:Unblock|Unblock an item:Unblock`, + dangerButtonText: $localize`:Block|Block an item:Block`, + listElements: this.toBulletPoints(websites, projects), + enableCancelButton: true, + onPrimaryButtonClick: async (close) => await block(close, false), + onDangerButtonClick: async (close) => await block(close, true), + }; + } else { + data = { + text: $localize`:Select websites again|No websites were selected so there is nothing to delete:Select the websites to block and try again.`, + title: $localize`:Nothing to block|Tried to block something, but there was nothing to delete:Nothing to block`, + primaryButtonText: $localize`:Ok|Accept or confirm:Ok`, + onPrimaryButtonClick: (close) => close(false), + }; + } + + return firstValueFrom( + this.dialog + .open(ConfirmDialogComponent, { + data, + restoreFocus: false, + }) + .afterClosed() + ); + } + + public block(websiteId: string, block: boolean) { + const errorBlocking = $localize`:Error while blocking|Error while blocking an item:Error while blocking`; + if (!websiteId) { + this.toastr.error(errorBlocking); + } + + let data: ConfirmDialogData = { + text: block + ? $localize`:Confirm block website|Confirmation message asking if the user wants to block the website:Do you really wish to block this website?` + : $localize`:Confirm unblock website|Confirmation message asking if the user wants to unblock the website:Do you really wish to unblock this website?`, + title: block + ? $localize`:Blocking website|Title of a page to block a website:Blocking website` + : $localize`:Unblocking website|Title of a page to unblock a website:Unblocking website`, + primaryButtonText: block ? $localize`:Block|Block an item:Block` : $localize`:Unblock|Unblock an item:Unblock`, + enableCancelButton: true, + onPrimaryButtonClick: async (close) => { + try { + await this.websitesService.block([websiteId], block); + this.toastr.success( + block + ? $localize`:Website blocked|Blocked a website:Website successfully blocked` + : $localize`:Website unblocked|Unblocked a website:Website successfully unblocked` + ); + + close(true); + } catch { + this.toastr.error(errorBlocking); + close(false); + } + }, + }; + + return firstValueFrom( + this.dialog + .open(ConfirmDialogComponent, { + data, + restoreFocus: false, + }) + .afterClosed() + ); + } + + private toBulletPoints(websites: Pick[], projects?: ProjectSummary[]) { + return websites.map((x) => { + const projectName = projects?.find((d) => d.id === x.projectId)?.name; + return projectName ? `${x.url} (${projectName})` : `${x.url}`; + }); + } +} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/websites.module.ts b/packages/frontend/stalker-app/src/app/modules/websites/websites.module.ts new file mode 100644 index 000000000..f8dd58f74 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/websites.module.ts @@ -0,0 +1,20 @@ +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; + +const routes: Routes = [ + { + path: '', + loadComponent: () => import('./list-websites/list-websites.component').then((m) => m.ListWebsitesComponent), + }, + { + path: ':id', + loadComponent: () => import('./view-website/view-website.component').then((m) => m.ViewWebsiteComponent), + }, +]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [], + providers: [], +}) +export class WebsitesListModule {} diff --git a/packages/frontend/stalker-app/src/app/shared/components/sidebar/sidebar.component.html b/packages/frontend/stalker-app/src/app/shared/components/sidebar/sidebar.component.html index 1949c7bda..2072aed3a 100644 --- a/packages/frontend/stalker-app/src/app/shared/components/sidebar/sidebar.component.html +++ b/packages/frontend/stalker-app/src/app/shared/components/sidebar/sidebar.component.html @@ -82,6 +82,22 @@

Pages

{{ ports }} } + + + web + @if (expanded) { + {{ websites }} + } + + implem private autocompleteFilter(value: string) { if (!value) return this.filterOptions?.filter((col) => col !== 'select'); - let filterValue = value.toLowerCase(); - const filterIsNegated = filterValue.length > 0 && filterValue[0]; - filterValue = filterIsNegated === '-' ? filterValue.slice(1) : filterValue; + let filterValue = value.toLowerCase().trimStart(); + const filterIsNegated = filterValue.length > 0 && filterValue[0] === '-'; + filterValue = filterIsNegated ? filterValue.slice(1) : filterValue; return this.filterOptions?.filter((col) => { const columnIncludesFilter = col.toLowerCase().includes(filterValue) && col !== 'select'; if (filterIsNegated) { From 9133f73f34ba772bd236dea9583faf041f870601 Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 24 Jun 2024 15:14:46 -0400 Subject: [PATCH 03/11] removing debugging log --- .../src/modules/findings/commands/JobFindings/website.handler.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/website.handler.ts b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/website.handler.ts index a912cf5cb..7c85e271e 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/website.handler.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/website.handler.ts @@ -34,7 +34,6 @@ export class WebsiteHandler extends JobFindingHandlerBase { } protected async executeCore(command: WebsiteCommand) { - console.log(command.finding); await this.websiteService.addWebsite( command.projectId, command.finding.ip, From 4f383c355c1d858166b9f5edaa6905f92b8162bb Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 24 Jun 2024 16:16:26 -0400 Subject: [PATCH 04/11] fixing syntax in website e2e --- .../modules/database/reporting/websites/website.e2e-spec.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts index 4e2bfd64d..762eb88df 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.e2e-spec.ts @@ -9,7 +9,7 @@ import { getReq, initTesting, patchReq, - putReq + putReq, } from '../../../../test/e2e.utils'; import { AppModule } from '../../../app.module'; import { Role } from '../../../auth/constants'; @@ -48,8 +48,6 @@ describe('Website Controller (e2e)', () => { await app.close(); }); - describe('Get websites', () => { - it('Should have proper authorizations (PUT /websites/:id/tags)', async () => { const success = await checkAuthorizations( testData, From d9a52e264c7af8f57d2df426e989b5d941c4469c Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 1 Jul 2024 16:50:00 -0400 Subject: [PATCH 05/11] first draft of website crwaling job --- jobs/job-base-images/python/Dockerfile | 36 ++-- .../built-in/code/code/WebsiteCrawlingJob.py | 171 ++++++++++++++++++ 2 files changed, 195 insertions(+), 12 deletions(-) create mode 100644 packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py diff --git a/jobs/job-base-images/python/Dockerfile b/jobs/job-base-images/python/Dockerfile index 756a570de..e3a785292 100644 --- a/jobs/job-base-images/python/Dockerfile +++ b/jobs/job-base-images/python/Dockerfile @@ -1,20 +1,32 @@ -FROM python:3.11.0-slim-bullseye +FROM python:3.12.4-slim-bullseye AS base RUN python -m pip install httpx[http2] RUN python -m pip install "poetry==1.3.2" COPY stalker_job_sdk /usr/src/stalker_job_sdk -RUN python -m pip install -e /usr/src/stalker_job_sdk \ - && apt-get update \ - && apt-get install -y nmap git make gcc libpcap0.8 \ - && mkdir -p /tools/masscan/ \ - && git clone https://github.com/robertdavidgraham/masscan /tools/masscan/ \ - && cd /tools/masscan/ \ - && make -j \ - && make install \ - && apt-get remove -y git make gcc \ - && apt-get autoremove -y \ - && rm -rf /tools/ +RUN python -m pip install -e /usr/src/stalker_job_sdk +RUN apt-get update && apt-get install -y nmap libpcap0.8 wget gnupg + +FROM base AS build +RUN uname -a +RUN apt-get install -y git make gcc zip curl + +# Masscan +RUN mkdir -p /tools/masscan/ +RUN git clone https://github.com/robertdavidgraham/masscan /tools/masscan/ +WORKDIR /tools/masscan +RUN make -j && make install + +# Katana +FROM projectdiscovery/katana:latest AS katana + +FROM base AS final +COPY --from=build /usr/bin/masscan /usr/bin/masscan +COPY --from=katana /usr/local/bin/katana /usr/bin/katana + +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && sh -c 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt update && apt install -y google-chrome-stable WORKDIR /usr/src/stalker-job diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py new file mode 100644 index 000000000..f094fb361 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py @@ -0,0 +1,171 @@ +import os +import typing +from json import loads +from subprocess import PIPE, Popen, run +from urllib.parse import urlparse + +from stalker_job_sdk import (JobStatus, TextField, WebsiteFinding, is_valid_ip, + is_valid_port, log_error, log_finding, log_info, + log_status, log_warning) + + +class WebsiteRequest: + method: str + endpoint: str + tag: str + attribute: str + source: str + +class WebsiteResponse: + status_code: int + headers: typing.Dict[str, str] + technologies: 'list[str]' + +class WebsiteFile: + timestamp: str + request: WebsiteRequest + response: WebsiteResponse + + def __init__(self, data: dict): + self.timestamp = data.get("timestamp") + self.request = WebsiteRequest() + req: dict = data.get("request") + self.request.attribute = req.get("attribute") + self.request.endpoint = req.get("endpoint") + self.request.method = req.get("method") + self.request.source = req.get("source") + self.request.tag = req.get("tag") + self.response = WebsiteResponse() + res: dict = data.get("response") + self.response.headers = res.get("headers") + self.response.status_code = res.get("status_code") + self.response.technologies = res.get("technologies") + + +def get_valid_args(): + """Gets the arguments from environment variables""" + target_ip: str = os.environ.get("targetIp") + port: int = int(os.environ.get("port")) + domain: str = os.environ.get("domainName") + path: str = os.environ.get("path") + ssl: str = os.environ.get("ssl") + max_depth: int = int(os.environ.get("maxDepth")) + crawl_duration_seconds: int = int(os.environ.get("crawlDurationSeconds")) + concurrency : int = int(os.environ.get("fetcherConcurrency")) + parallelism : int = int(os.environ.get("inputParallelism")) + + extra_katana_option: str = os.environ.get("extraOptions") + + if not is_valid_ip(target_ip): + log_error(f"targetIp parameter is invalid: {target_ip}") + log_status(JobStatus.FAILED) + exit() + + if not is_valid_port(port): + log_error(f"port parameter is invalid: {str(port)}") + log_status(JobStatus.FAILED) + exit() + + if max_depth <= 0: + log_error(f"maxDepth parameter is invalid: {str(max_depth)}") + log_status(JobStatus.FAILED) + exit() + + if crawl_duration_seconds <= 0: + log_error(f"crawlDurationSeconds parameter is invalid: {str(crawl_duration_seconds)}") + log_status(JobStatus.FAILED) + exit() + + if concurrency <= 0: + log_error(f"fetcherConcurrency parameter is invalid: {str(concurrency)}") + log_status(JobStatus.FAILED) + exit() + + if parallelism <= 0: + log_error(f"inputParallelism parameter is invalid: {str(parallelism)}") + log_status(JobStatus.FAILED) + exit() + + return target_ip, port, domain, path, ssl, max_depth, crawl_duration_seconds, concurrency, parallelism, extra_katana_option + + +def emit_file_finding(file: WebsiteFile, domain: str, ip: str, port: int, path: str, ssl: bool): + fields = [] + + if file.response.status_code: + fields.append(TextField("statusCode", "Status Code", file.response.status_code)) + + if file.request.endpoint: + endpoint = urlparse(file.request.endpoint).path + fields.append(TextField("endpoint", "Endpoint", endpoint)) + + if file.request.method: + fields.append(TextField("method", "Method", file.request.method)) + + if file.request.tag: + fields.append(TextField("tag", "Tag", file.request.tag)) + + if file.request.attribute: + fields.append(TextField("attribute", "Attribute", file.request.attribute)) + + if file.request.source: + fields.append(TextField("source", "Source", file.request.source)) + + log_finding( + WebsiteFinding( + "WebsitePath", ip, port, domain, path, ssl, f"Website path", fields + ) + ) + +def emit_technology_findings(technologies: 'list[str]', domain: str, ip: str, port: int, path: str, ssl: bool): + for tech in technologies: + log_finding( + WebsiteFinding( + "WebsiteTechnology", ip, port, domain, path, ssl, f"Website path", [TextField("technology", "Technology", tech)] + ) + ) + +def build_url(ip: str, port: int, domain: str, path: str, ssl: bool): + url = "https://" if ssl else "http://" + url += domain if domain else ip + url += f":{str(port)}" if port != 80 and port != 443 else "" + url += path + return url + +def main(): + target_ip, port, domain, path, ssl, max_depth, crawl_duration_seconds, concurrency, parallelism, extra_options = get_valid_args() + url = build_url(target_ip, port, domain, path, ssl) + + katana_str: str = f"katana -u {url} -d {max_depth} -ct {crawl_duration_seconds} -c {str(concurrency)} -p {str(parallelism)} {extra_options}" + log_info(f'Start of crawling: {katana_str}') + + # katana -u https://example.com -silent -d 3 -ct 3600 -jc -kf all -timeout 3 -duc -j -or -ob -c 10 -p 10 + technologies: 'set[str]' = set() + with Popen(katana_str, stdout=PIPE, universal_newlines=True, shell=True) as katana_process: + + for line in katana_process.stdout: + file = '' + try: + file: WebsiteFile = WebsiteFile(loads(line)) + if file: + if file.response.technologies: + technologies.update(file.response.technologies) + + if file.response.status_code == 404: + continue + + emit_file_finding(file, domain, target_ip, port, path, ssl) + except Exception as err: + log_warning(err) + continue + + emit_technology_findings(technologies, domain, target_ip, port, path, ssl) + +try: + main() + log_status(JobStatus.SUCCESS) +except Exception as err: + log_error("An unexpected error occured") + log_error(err) + log_status(JobStatus.FAILED) + exit() \ No newline at end of file From e66c4fbf6241fbd0d07e6245ae4bca3bd298f5c8 Mon Sep 17 00:00:00 2001 From: lm-sec Date: Wed, 3 Jul 2024 15:00:12 -0400 Subject: [PATCH 06/11] adding website crawling, more tests required --- docs/docs/concepts/findings.md | 44 ++++++++- docs/docs/concepts/jobs.md | 29 +++++- docs/docs/concepts/subscriptions.md | 8 +- jobs/job-base-images/python/Dockerfile | 15 ++- .../stalker_job_sdk/__init__.py | 5 +- .../built-in/code/code/HttpGetTemplate.py | 9 +- .../built-in/code/WebsiteCrawlingJob.yaml | 35 +++++++ .../built-in/code/code/BannerGrabbingJob.py | 6 +- .../code/code/DomainNameResolvingJob.py | 2 +- .../code/code/TcpIpRangeScanningJob.py | 10 +- .../built-in/code/code/TcpPortScanningJob.py | 10 +- .../built-in/code/code/WebsiteCrawlingJob.py | 9 +- .../database/jobs/models/custom-job.model.ts | 3 +- .../reporting/websites/website.service.ts | 96 ++++++++++++++----- .../built-in/website-crawling.yml | 26 +++++ .../commands/JobFindings/custom.handler.ts | 79 ++++++++++----- .../commands/JobFindings/website.handler.ts | 2 +- .../src/modules/findings/findings.service.ts | 12 ++- .../JobTemplates/PythonCustomJobTemplate.cs | 6 +- .../view-website/view-website.component.html | 2 +- 20 files changed, 312 insertions(+), 96 deletions(-) create mode 100644 packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/WebsiteCrawlingJob.yaml create mode 100644 packages/backend/jobs-manager/service/src/modules/database/subscriptions/event-subscriptions/built-in/website-crawling.yml diff --git a/docs/docs/concepts/findings.md b/docs/docs/concepts/findings.md index f28199b66..7e3a713f0 100644 --- a/docs/docs/concepts/findings.md +++ b/docs/docs/concepts/findings.md @@ -21,7 +21,7 @@ The finding object must contain the `type` field. Here is a list of available ty | [IpRangeFinding](#iprangefinding) | Creates a new IP range. | | [HostnameIpFinding](#hostnameipfinding) | Creates a new host, attaches it to a given domain. | | [PortFinding](#portfinding) | Creates a new port, attaches it to the given host. | -| [WebsiteFinding](#websiteFinding) | Creates a new website, with the proper host, domain and port | +| [WebsiteFinding](#websitefinding) | Creates a new website, with the proper host, domain and port | | [CustomFinding](#customfinding) | Attaches custom finding data to a given entity. | | [PortServiceFinding](#portservicefinding) | Fills the `service` field of a port. | @@ -218,6 +218,7 @@ and attaches it to the given host. | `port` | The port number | | `domain` | The domain on which the website is hosted, can be empty | | `path` | The folder path, defaults to `/` | +| `ssl` | True if the website is protected by encryption | Example: @@ -227,7 +228,7 @@ Example: "key": "WebsiteFinding", "ip": "1.2.3.4", "port": 80, - "domain": "example.com", + "domainName": "example.com", "path": "/", "ssl": false } @@ -308,7 +309,7 @@ port = 80 ip = "0.0.0.0" log_finding( PortFinding( - "HttpServerCheck", ip, port, "tcp", "This port runs an HTTP server" + "PortFunFact", ip, port, "tcp", "This is a fun fact about a port" ) ) ``` @@ -442,3 +443,40 @@ We would create the following three websites: | dev.example.com | 1.2.3.4 | 443 | `/` | That way, a website at `dev.example.com`, which may be different than the one at `example.com`, will be found. The same goes for the website through direct IP access. + +## WebsitePathFinding + +A `WebsitePathFinding` is type of `CustomFinding` that fills a website's `paths` database field with the `endpoint` text field label. It will then be shown in the interface as the website's site map. + +| Field | Description | +| -------- | ------------------------------------------------------------- | +| `domain` | The website's domain | +| `ip` | The website's ip | +| `port` | The website's port number | +| `path` | The website's path | +| `fields` | A list of [fields](#dynamic-fields). Must include `endpoint`. | + +Using the python SDK, you can emit this finding with the following code. + +```python +from stalker_job_sdk import PortFinding, log_finding, TextField + +ip = '1.2.3.4' +domain = 'example.com' +port = 443 +path = '/' +ssl = True +endpoint = '/example/endpoint.html' + +fields = [ + TextField("endpoint", "Enspoint", endpoint) +] + +log_finding( + WebsiteFinding( + "WebsitePathFinding", ip, port, domain, path, ssl, f"Website path", fields + ) +) +``` + +Upon receiving this finding, the backend will populate the proper website's path with the `endpoint` data. diff --git a/docs/docs/concepts/jobs.md b/docs/docs/concepts/jobs.md index 22181a834..2f77f9cd3 100644 --- a/docs/docs/concepts/jobs.md +++ b/docs/docs/concepts/jobs.md @@ -25,7 +25,8 @@ subscription would need to be adapted to the new name. | [DomainNameResolvingJob](#domainnameresolvingjob) | Resolves a domain name to an IP address | | [TcpPortScanningJob](#tcpportscanningjob) | Scans the tcp ports of an IP address | | [TcpIpRangeScanningJob](#tcpiprangescanningjob) | Scan an IP range for open ports | -| [BannerGrabbingJob](#httpservercheckjob) | Identifies the service running on a port | +| [BannerGrabbingJob](#bannergrabbingjob) | Identifies the service running on a port | +| [WebsiteCrawlingJob](#websitecrawlingjob) | Crawls a website for its valid endpoints | ### DomainNameResolvingJob @@ -90,10 +91,34 @@ Identifies the service running on a port and grabs the banner. It may occasional | ------------- | -------- | -------------------------------------------------- | | targetIp | string | The IP address to check | | ports | number[] | The ports to check for a service | -| nmapOptions | | A long string containing the options given to nmap | +| nmapOptions | string | A long string containing the options given to nmap | **Possible findings generated :** - PortServiceFinding - HostnameIpFinding - OperatingSystemFinding + +### WebsiteCrawlingJob + +Crawls a website for its differents valid endpoints. It can also find website technology information. + +**Input variables:** + +| Variable Name | Type | Value description | +| -------------------- | ------ | --------------------------------------------- | +| targetIp | string | The website's IP address | +| port | number | The website's port | +| domainName | string | The website's domain name | +| path | string | The website's base path | +| ssl | bool | If the website is https | +| maxDepth | number | The depth to crawl | +| crawlDurationSeconds | number | The max amount of time to crawl in seconds | +| fetcherConcurrency | number | The number of concurrent fetchers to get data | +| inputParallelism | number | The number of concurrent inputs pprocessor | +| extraOptions | string | Katana extra options to adapt execution | + +**Possible findings generated :** + +- WebsitePathFinding +- WebsiteTechnologyFinding diff --git a/docs/docs/concepts/subscriptions.md b/docs/docs/concepts/subscriptions.md index 0d6998920..de42443f1 100644 --- a/docs/docs/concepts/subscriptions.md +++ b/docs/docs/concepts/subscriptions.md @@ -215,16 +215,12 @@ An event subscription can contain these main elements : - `operator` : The operator to compare the two operands. - `rhs` : The right-hand side operand. - - - - #### Event Subscription Dynamic Input You can add dynamic input to an event subscription either by referencing a finding's fields, or by injecting a secret. -You can reference a Finding's output variable by name in a Job parameter's value or in a condition's operand using the following syntax: -`${parameterName}`. The variable name is case insensitive. +You can reference a Finding's output variable by name in a Job parameter's value or in a condition's operand using the following syntax: +`${parameterName}`. The variable name is case insensitive. In a finding, you can find [dynamic fields](/docs/concepts/findings#dynamic-fields) in the `fields` array. The text based dynamic fields' values can be injected in the same way as a regular field, with the `${parameterName}` syntax. Simply reference the `key` part of a dynamic field as the variable name, and its `data` will be injected. diff --git a/jobs/job-base-images/python/Dockerfile b/jobs/job-base-images/python/Dockerfile index e3a785292..1608a7145 100644 --- a/jobs/job-base-images/python/Dockerfile +++ b/jobs/job-base-images/python/Dockerfile @@ -6,10 +6,9 @@ RUN python -m pip install "poetry==1.3.2" COPY stalker_job_sdk /usr/src/stalker_job_sdk RUN python -m pip install -e /usr/src/stalker_job_sdk -RUN apt-get update && apt-get install -y nmap libpcap0.8 wget gnupg +RUN apt-get update && apt-get install -y nmap libpcap0.8 wget gnupg libc6 FROM base AS build -RUN uname -a RUN apt-get install -y git make gcc zip curl # Masscan @@ -19,11 +18,19 @@ WORKDIR /tools/masscan RUN make -j && make install # Katana -FROM projectdiscovery/katana:latest AS katana +FROM golang:1.22.4-bullseye AS katana + +RUN apt-get update && apt-get install -y git gcc musl-dev +RUN mkdir -p /tools/katana/ +RUN git clone https://github.com/projectdiscovery/katana.git /tools/katana +WORKDIR /tools/katana +RUN go mod download +RUN go build ./cmd/katana FROM base AS final + COPY --from=build /usr/bin/masscan /usr/bin/masscan -COPY --from=katana /usr/local/bin/katana /usr/bin/katana +COPY --from=katana /tools/katana/katana /usr/bin/katana RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && sh -c 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ diff --git a/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py b/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py index 7909512fa..38e373217 100644 --- a/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py +++ b/jobs/job-base-images/python/stalker_job_sdk/stalker_job_sdk/__init__.py @@ -80,7 +80,7 @@ def __init__( key: str, ip: str, port: int, - domain: str, + domainName: str, path: str, ssl: bool = None, name: str = None, @@ -90,7 +90,8 @@ def __init__( super().__init__(key, type, name, fields) self.ip = ip self.port = port - self.domain = domain + self.domainName = domainName + self.protocol = 'tcp' self.path = path self.ssl = ssl diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-job-templates/built-in/code/code/HttpGetTemplate.py b/packages/backend/jobs-manager/service/src/modules/database/custom-job-templates/built-in/code/code/HttpGetTemplate.py index c5b8b0f26..acd241caa 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/custom-job-templates/built-in/code/code/HttpGetTemplate.py +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-job-templates/built-in/code/code/HttpGetTemplate.py @@ -1,5 +1,4 @@ import os -import random import httpx from stalker_job_sdk import (JobStatus, PortFinding, TextField, is_valid_ip, @@ -9,10 +8,10 @@ def get_args(): """Gets the arguments from environment variables""" - target_ip: str = os.environ["targetIp"] - port = int(os.environ["port"]) - domain = os.environ["domainName"] # DOMAIN should resolve to TARGET_IP - path = os.environ["path"] # Http server file path to GET + target_ip: str = os.environ.get("targetIp") + port = int(os.environ.get("port")) + domain = os.environ.get("domainName") # DOMAIN should resolve to TARGET_IP + path = os.environ.get("path") # Http server file path to GET if not path or len(path) == 0: path = '/' diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/WebsiteCrawlingJob.yaml b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/WebsiteCrawlingJob.yaml new file mode 100644 index 000000000..34b5e7590 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/WebsiteCrawlingJob.yaml @@ -0,0 +1,35 @@ +name: WebsiteCrawlingJob +type: code +language: python +parameters: + - name: domainName + type: string + - name: targetIp + type: string + - name: port + type: number + default: 443 + - name: path + type: string + default: "/" + - name: ssl + type: boolean + default: true + - name: maxDepth + type: number + default: 3 + - name: crawlDurationSeconds + type: number + default: 1800 + - name: fetcherConcurrency + type: number + default: 10 + - name: inputParallelism + type: number + default: 10 + - name: extraOptions + value: "-jc -kf all -duc -j -or -ob -silent" +jobPodConfigName: M +codeFilePath: code/WebsiteCrawlingJob.py +templateOrdering: '/built-in/python' + \ No newline at end of file diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/BannerGrabbingJob.py b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/BannerGrabbingJob.py index 2858fb688..4c55551b3 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/BannerGrabbingJob.py +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/BannerGrabbingJob.py @@ -19,9 +19,9 @@ class PortInfo: def get_valid_args(): """Gets the arguments from environment variables""" - target_ip: str = os.environ["targetIp"] - ports_str: str = os.environ["ports"] - nmap_options: str = os.environ["nmapOptions"] + target_ip: str = os.environ.get("targetIp") + ports_str: str = os.environ.get("ports") + nmap_options: str = os.environ.get("nmapOptions") ports_list:list = [] ports_set: set = set() diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/DomainNameResolvingJob.py b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/DomainNameResolvingJob.py index e0209e361..2286a770a 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/DomainNameResolvingJob.py +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/DomainNameResolvingJob.py @@ -3,7 +3,7 @@ from stalker_job_sdk import DomainFinding, JobStatus, log_finding, log_status -hostname = os.environ["domainName"] +hostname = os.environ.get("domainName") data = socket.gethostbyname_ex(hostname) ipx = data[2] diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpIpRangeScanningJob.py b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpIpRangeScanningJob.py index 2be7cf52c..c2f4e1e00 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpIpRangeScanningJob.py +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpIpRangeScanningJob.py @@ -21,12 +21,12 @@ def validate_port(port: int, name: str): exit() def main(): - TARGET_IP: str = environ["targetIp"] # Start of ip range - TARGET_MASK: int = int(environ["targetMask"]) # mask (ex: /24) - RATE: int = int(environ["rate"]) # number of threads to do the requests - PORT_MIN: int = int(environ["portMin"]) # expects a number (0 < p1 < 65535) + TARGET_IP: str = environ.get("targetIp") # Start of ip range + TARGET_MASK: int = int(environ.get("targetMask")) # mask (ex: /24) + RATE: int = int(environ.get("rate")) # number of threads to do the requests + PORT_MIN: int = int(environ.get("portMin")) # expects a number (0 < p1 < 65535) PORT_MAX: int = int( - environ["portMax"] + environ.get("portMax") ) # expects a number (0 < p2 <= 65535 and p2 > p1) PORTS = environ[ diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpPortScanningJob.py b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpPortScanningJob.py index 0c3411211..1db544b85 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpPortScanningJob.py +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/TcpPortScanningJob.py @@ -45,14 +45,14 @@ def run(self): self.open_ports.append(p) -TARGET_IP: str = os.environ["targetIp"] # IP to scan -THREADS: int = int(os.environ["threads"]) # number of threads to do the requests +TARGET_IP: str = os.environ.get("targetIp") # IP to scan +THREADS: int = int(os.environ.get("threads")) # number of threads to do the requests SOCKET_TIMEOUT: float = float( - os.environ["socketTimeoutSeconds"] + os.environ.get("socketTimeoutSeconds") ) # time in seconds to wait for socket -PORT_MIN: int = int(os.environ["portMin"]) # expects a number (0 < p1 < 65535) +PORT_MIN: int = int(os.environ.get("portMin")) # expects a number (0 < p1 < 65535) PORT_MAX: int = int( - os.environ["portMax"] + os.environ.get("portMax") ) # expects a number (0 < p2 <= 65535 and p2 > p1) PORTS = os.environ[ diff --git a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py index f094fb361..0beb51446 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py +++ b/packages/backend/jobs-manager/service/src/modules/database/custom-jobs/built-in/code/code/WebsiteCrawlingJob.py @@ -113,7 +113,7 @@ def emit_file_finding(file: WebsiteFile, domain: str, ip: str, port: int, path: log_finding( WebsiteFinding( - "WebsitePath", ip, port, domain, path, ssl, f"Website path", fields + "WebsitePathFinding", ip, port, domain, path, ssl, f"Website path", fields ) ) @@ -121,7 +121,7 @@ def emit_technology_findings(technologies: 'list[str]', domain: str, ip: str, po for tech in technologies: log_finding( WebsiteFinding( - "WebsiteTechnology", ip, port, domain, path, ssl, f"Website path", [TextField("technology", "Technology", tech)] + "WebsiteTechnologyFinding", ip, port, domain, path, ssl, f"Technology", [TextField("technology", "Technology", tech)] ) ) @@ -141,7 +141,7 @@ def main(): # katana -u https://example.com -silent -d 3 -ct 3600 -jc -kf all -timeout 3 -duc -j -or -ob -c 10 -p 10 technologies: 'set[str]' = set() - with Popen(katana_str, stdout=PIPE, universal_newlines=True, shell=True) as katana_process: + with Popen(katana_str, stdout=PIPE, stderr=PIPE, universal_newlines=True, shell=True) as katana_process: for line in katana_process.stdout: file = '' @@ -158,6 +158,9 @@ def main(): except Exception as err: log_warning(err) continue + + for line in katana_process.stderr: + log_error(line) emit_technology_findings(technologies, domain, target_ip, port, path, ssl) diff --git a/packages/backend/jobs-manager/service/src/modules/database/jobs/models/custom-job.model.ts b/packages/backend/jobs-manager/service/src/modules/database/jobs/models/custom-job.model.ts index 1966e2f32..f59fa73e4 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/jobs/models/custom-job.model.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/jobs/models/custom-job.model.ts @@ -1,4 +1,5 @@ import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose'; +import { isNullOrUndefined } from '@typegoose/typegoose/lib/internal/utils'; import { isArray, isEmpty, @@ -142,7 +143,7 @@ export class CustomJob { !isString(param.name) || !environmentVariableRegex.test(param.name) || environmentVariableConflict.some((v) => v === param.name) || - isEmpty(param.value), + isNullOrUndefined(param.value), ) ) { throw new JobParameterValueException( diff --git a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts index 9007a5cf9..6531465f1 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/reporting/websites/website.service.ts @@ -11,11 +11,18 @@ import { CorrelationKeyUtils } from '../correlation.utils'; import { Domain } from '../domain/domain.model'; import { DomainSummary } from '../domain/domain.summary'; import { Host } from '../host/host.model'; -import { Port } from '../port/port.model'; +import { Port, PortDocument } from '../port/port.model'; import { WebsiteFilterModel } from './website-filter.model'; import { BatchEditWebsitesDto } from './website.dto'; import { Website, WebsiteDocument } from './website.model'; +interface ExtendedWebsiteSearchQuery { + projectIdObj: Types.ObjectId; + existingDomainSummary: DomainSummary | undefined; + existingPort: PortDocument; + searchQuery: FilterQuery; +} + @Injectable() export class WebsiteService { private logger = new Logger(WebsiteService.name); @@ -29,14 +36,13 @@ export class WebsiteService { private tagsService: TagsService, ) {} - public async addWebsite( + private async buildExtendedSearchQuery( projectId: string, ip: string, port: number, domain: string = undefined, path: string = '/', - ssl: boolean = undefined, - ) { + ): Promise { const projectIdObj = new Types.ObjectId(projectId); const existingPort = await this.portModel.findOne({ 'host.ip': { $eq: ip }, @@ -78,32 +84,54 @@ export class WebsiteService { existingDomainSummary = undefined; } - // Search for a website with the proper port id, domain and path. - // domain may or may not exist, it depends if it was found in the domains collection - const searchQuery: FilterQuery = { - 'port.id': { $eq: existingPort._id }, - 'domain.id': { - $eq: existingDomainSummary ? existingDomainSummary.id : null, + return { + projectIdObj: projectIdObj, + existingDomainSummary: existingDomainSummary, + existingPort: existingPort, + searchQuery: { + 'port.id': { $eq: existingPort._id }, + 'domain.id': { + $eq: existingDomainSummary ? existingDomainSummary.id : null, + }, + path: { $eq: path }, }, - path: { $eq: path }, }; + } - return this.websiteModel.findOneAndUpdate( - searchQuery, + public async addWebsite( + projectId: string, + ip: string, + port: number, + domain: string = undefined, + path: string = '/', + ssl: boolean = undefined, + ) { + // Search for a website with the proper port id, domain and path. + // domain may or may not exist, it depends if it was found in the domains collection + const extendedSearchQuery: ExtendedWebsiteSearchQuery = + await this.buildExtendedSearchQuery(projectId, ip, port, domain, path); + + return await this.websiteModel.findOneAndUpdate( + extendedSearchQuery.searchQuery, { $set: { lastSeen: Date.now(), ssl: ssl ?? null }, $setOnInsert: { - host: existingPort.host, - domain: existingDomainSummary ?? null, - port: { id: existingPort._id, port: existingPort.port }, + host: extendedSearchQuery.existingPort.host, + domain: extendedSearchQuery.existingDomainSummary ?? null, + port: { + id: extendedSearchQuery.existingPort._id, + port: extendedSearchQuery.existingPort.port, + }, path: path, - sitemap: ['/'], - projectId: projectIdObj, + sitemap: [], + projectId: extendedSearchQuery.projectIdObj, correlationKey: CorrelationKeyUtils.websiteCorrelationKey( projectId, ip, port, - existingDomainSummary ? existingDomainSummary.name : '', + extendedSearchQuery.existingDomainSummary + ? extendedSearchQuery.existingDomainSummary.name + : '', path, ), }, @@ -112,6 +140,29 @@ export class WebsiteService { ); } + public async addPathToWebsite( + pathToAdd: string, + projectId: string, + ip: string, + port: number, + domain: string = undefined, + path: string = '/', + ) { + const extendedSearchQuery = await this.buildExtendedSearchQuery( + projectId, + ip, + port, + domain, + path, + ); + + return await this.websiteModel.findOneAndUpdate( + extendedSearchQuery.searchQuery, + { $addToSet: { sitemap: pathToAdd } }, + { new: true }, + ); + } + /** * This method creates a WebsiteFinding for each domain linked to the host, * as well as one for the host's direct ip port access, without a domain. @@ -142,27 +193,28 @@ export class WebsiteService { throw new HttpNotFoundException(); } - const websiteFindingBase: Omit = { + const websiteFindingBase: Omit = { type: 'WebsiteFinding', key: 'WebsiteFinding', ip: ip, path: path, port: port, fields: [], + protocol: 'tcp', ssl: ssl, }; // We will create a website finding let findings: WebsiteFinding[] = [ { - domain: '', + domainName: '', ...websiteFindingBase, }, ]; for (const domainSummary of host.domains) { findings.push({ - domain: domainSummary.name, + domainName: domainSummary.name, ...websiteFindingBase, }); diff --git a/packages/backend/jobs-manager/service/src/modules/database/subscriptions/event-subscriptions/built-in/website-crawling.yml b/packages/backend/jobs-manager/service/src/modules/database/subscriptions/event-subscriptions/built-in/website-crawling.yml new file mode 100644 index 000000000..10fbae687 --- /dev/null +++ b/packages/backend/jobs-manager/service/src/modules/database/subscriptions/event-subscriptions/built-in/website-crawling.yml @@ -0,0 +1,26 @@ +name: Website crawling +finding: WebsiteFinding +cooldown: 86400 +job: + name: WebsiteCrawlingJob + parameters: + - name: domainName + value: ${domainName} + - name: targetIp + value: ${ip} + - name: port + value: ${port} + - name: path + value: ${path} + - name: ssl + value: ${ssl} + - name: maxDepth + value: 3 + - name: crawlDurationSeconds + value: 1800 + - name: fetcherConcurrency + value: 10 + - name: inputParallelism + value: 10 + - name: extraOptions + value: "-jc -kf all -duc -j -or -ob -silent" diff --git a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts index df8e0ff1e..1eac94a34 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts @@ -38,38 +38,65 @@ export class CustomFindingHandler extends JobFindingHandlerBase { command.projectId, command.finding.ip, command.finding.port, - command.finding.domain, + command.finding.domainName, command.finding.path, command.finding.ssl || command.finding.ssl === false ? command.finding.ssl diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts index 639f0bcf8..ee78bc08f 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts @@ -57,9 +57,10 @@ export class WebsiteFinding extends FindingBase { key: 'WebsiteFinding'; ip: string; port: number; - domain: string = ''; + domainName: string = ''; path: string = '/'; ssl?: boolean; + protocol: 'tcp' = 'tcp'; } export class CreateCustomFinding extends FindingBase { @@ -70,6 +71,7 @@ export class CreateCustomFinding extends FindingBase { port?: number; protocol?: 'tcp' | 'udp'; name: string; + path?: string; } export class JobStatusFinding extends FindingBase { @@ -191,6 +193,8 @@ export class FindingsService { dto.ip, dto.port, dto.protocol, + null, + dto.path, ); const finding: CustomFinding = { @@ -358,12 +362,12 @@ export class FindingsService { case 'WebsiteFinding': finding.correlationKey = CorrelationKeyUtils.generateCorrelationKey( projectId, - finding.domain, + finding.domainName ?? '', finding.ip, finding.port, 'tcp', null, - finding.path, + finding.path ?? '/', ); this.commandBus.execute( new WebsiteCommand(jobId, projectId, WebsiteCommand.name, finding), @@ -377,6 +381,8 @@ export class FindingsService { finding.ip, finding.port, finding.protocol, + null, + finding.path, ); this.commandBus.execute( new CustomFindingCommand( diff --git a/packages/backend/orchestrator/service/Orchestrator/Jobs/JobTemplates/PythonCustomJobTemplate.cs b/packages/backend/orchestrator/service/Orchestrator/Jobs/JobTemplates/PythonCustomJobTemplate.cs index 9ec6181a3..b13c1c983 100644 --- a/packages/backend/orchestrator/service/Orchestrator/Jobs/JobTemplates/PythonCustomJobTemplate.cs +++ b/packages/backend/orchestrator/service/Orchestrator/Jobs/JobTemplates/PythonCustomJobTemplate.cs @@ -14,7 +14,7 @@ public PythonCustomJobTemplate(string? id, IConfiguration config, JobParameter[] { foreach (var param in jobParameters!) { - if (param.Name.IsNullOrEmpty() || param.Value.IsNullOrEmpty()) continue; + if (param.Name.IsNullOrEmpty() || param.Value == null) continue; EnvironmentVariable[param.Name!] = CryptoUtils.Decrypt(param.Value!); } } @@ -30,8 +30,8 @@ public PythonCustomJobTemplate(string? id, IConfiguration config, JobParameter[] if (jobPodMemoryKbLimit.HasValue && jobPodMemoryKbLimit > 0) { this.MemoryKiloBytesLimit = jobPodMemoryKbLimit; - } - + } + int? timeout = config.GetSection("Jobs").GetSection("CustomJobs").GetValue("Timeout"); if (timeout == null) throw new NullReferenceException("Setting Timeout is missing."); diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html index f9312b304..49e0285f4 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html @@ -20,7 +20,7 @@

- +
@if (wsite.blocked) { From fcce3978355ae3543d95e1dd45209e9218672d77 Mon Sep 17 00:00:00 2001 From: lm-sec Date: Fri, 5 Jul 2024 12:54:53 -0400 Subject: [PATCH 07/11] enhanced UI for websites --- .../commands/JobFindings/custom.handler.ts | 13 +- .../src/modules/findings/finding.dto.ts | 19 +++ .../modules/findings/findings.constants.ts | 7 + .../modules/findings/findings.controller.ts | 25 +-- .../src/modules/findings/findings.service.ts | 25 +++ .../src/app/api/findings/findings.service.ts | 28 ++- .../src/app/api/websites/websites.service.ts | 1 + .../view-domain/view-domain.component.scss | 2 +- .../findings-list/findings-list.component.ts | 11 +- .../hosts/view-host/view-host.component.scss | 2 +- .../ports/view-port/view-port.component.scss | 2 +- .../view-website/view-website.component.html | 127 +++++++++++++- .../view-website/view-website.component.scss | 159 +++++++++++++++++- .../view-website/view-website.component.ts | 32 +++- .../src/app/shared/shared.module.ts | 3 + .../app/shared/types/finding/finding.type.ts | 1 + .../status-code-pill-tag.component.ts | 117 +++++++++++++ 17 files changed, 548 insertions(+), 26 deletions(-) create mode 100644 packages/backend/jobs-manager/service/src/modules/findings/findings.constants.ts create mode 100644 packages/frontend/stalker-app/src/app/shared/widget/pill-tag/status-code-pill-tag.component.ts diff --git a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts index 1eac94a34..4965bc2ad 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/commands/JobFindings/custom.handler.ts @@ -9,6 +9,7 @@ import { WebsiteService } from '../../../database/reporting/websites/website.ser import { SecretsService } from '../../../database/secrets/secrets.service'; import { EventSubscriptionsService } from '../../../database/subscriptions/event-subscriptions/event-subscriptions.service'; import { SubscriptionTriggersService } from '../../../database/subscriptions/subscription-triggers/subscription-triggers.service'; +import { CustomFindingsConstants } from '../../findings.constants'; import { FindingsService } from '../../findings.service'; import { JobFindingHandlerBase } from '../job-findings-handler-base'; import { CustomFindingCommand } from './custom.command'; @@ -42,7 +43,8 @@ export class CustomFindingHandler extends JobFindingHandlerBase> { - if (dto.target == null || dto.target.trim() === '') { - throw new BadRequestException('Must provide a target.'); - } - return await this.findingsService.getAll( dto.target, +dto.page, +dto.pageSize, + dto.filterFinding, + ); + } + + @Get('endpoint') + async getLatestEndpoint( + @Query() dto: WebsiteEndpointFindingDto, + ): Promise { + return await this.findingsService.getLatestWebsiteEndpoint( + dto.target, + dto.endpoint, ); } } diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts index ee78bc08f..b5e077f9b 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts @@ -20,6 +20,7 @@ import { HostnameIpCommand } from './commands/JobFindings/hostname-ip.command'; import { PortCommand } from './commands/JobFindings/port.command'; import { WebsiteCommand } from './commands/JobFindings/website.command'; import { CustomFindingFieldDto } from './finding.dto'; +import { CustomFindingsConstants } from './findings.constants'; export type Finding = | HostnameIpFinding @@ -134,6 +135,7 @@ export class FindingsService { target: string, page: number, pageSize: number, + filterFindings: string[] = [], ): Promise> { if (page < 1) throw new HttpBadRequestException('Page starts at 1.'); @@ -145,6 +147,10 @@ export class FindingsService { }; } + if (filterFindings.length) { + filters.key = { $nin: filterFindings }; + } + const items = await this.findingModel .find(filters) .sort({ @@ -161,6 +167,25 @@ export class FindingsService { }; } + public async getLatestWebsiteEndpoint( + correlationKey: string, + endpoint: string, + ) { + return await this.findingModel + .findOne({ + correlationKey: { $eq: correlationKey }, + key: CustomFindingsConstants.WebsitePathFinding, + fields: { + $elemMatch: { + type: 'text', + key: CustomFindingsConstants.WebsiteEndpointFieldKey, + data: endpoint, + }, + }, + }) + .sort({ _id: 'ascending' }); + } + /** * Saves the given finding. */ diff --git a/packages/frontend/stalker-app/src/app/api/findings/findings.service.ts b/packages/frontend/stalker-app/src/app/api/findings/findings.service.ts index a858920c9..52f3edc8f 100644 --- a/packages/frontend/stalker-app/src/app/api/findings/findings.service.ts +++ b/packages/frontend/stalker-app/src/app/api/findings/findings.service.ts @@ -1,6 +1,6 @@ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; -import { Observable, tap } from 'rxjs'; +import { map, Observable, tap } from 'rxjs'; import { environment } from '../../../environments/environment'; import { CustomFinding } from '../../shared/types/finding/finding.type'; import { Page } from '../../shared/types/page.type'; @@ -11,9 +11,31 @@ import { Page } from '../../shared/types/page.type'; export class FindingsService { constructor(private http: HttpClient) {} - public getFindings(target: string | undefined = undefined, page = 1, pageSize = 25): Observable> { + public getFindings( + target: string | undefined = undefined, + page = 1, + pageSize = 25, + filterFindingKeys: string[] + ): Observable> { + let url = `${environment.fmUrl}/findings?target=${target}&page=${page}&pageSize=${pageSize}`; + for (let f of filterFindingKeys) { + url += `&filterFinding[]=${encodeURIComponent(f)}`; + } + return this.http - .get>(`${environment.fmUrl}/findings?target=${target}&page=${page}&pageSize=${pageSize}`) + .get>(url) .pipe(tap((x) => (x.items = x.items.map((i) => ({ ...i, created: new Date(i.created) }))))); } + + public getLatestWebsiteEndpoint(target: string, endpoint: string): Observable { + return this.http + .get( + `${environment.fmUrl}/findings/endpoint?target=${target}&endpoint=${encodeURIComponent(endpoint)}` + ) + .pipe( + map((x) => { + return { ...x, created: new Date(x.created) }; + }) + ); + } } diff --git a/packages/frontend/stalker-app/src/app/api/websites/websites.service.ts b/packages/frontend/stalker-app/src/app/api/websites/websites.service.ts index a703b9500..b69a1a9be 100644 --- a/packages/frontend/stalker-app/src/app/api/websites/websites.service.ts +++ b/packages/frontend/stalker-app/src/app/api/websites/websites.service.ts @@ -69,6 +69,7 @@ export class WebsitesService { website.ssl ), ...website, + sitemap: website.sitemap.sort(), }; }) ); diff --git a/packages/frontend/stalker-app/src/app/modules/domains/view-domain/view-domain.component.scss b/packages/frontend/stalker-app/src/app/modules/domains/view-domain/view-domain.component.scss index cede781b3..b9524652f 100644 --- a/packages/frontend/stalker-app/src/app/modules/domains/view-domain/view-domain.component.scss +++ b/packages/frontend/stalker-app/src/app/modules/domains/view-domain/view-domain.component.scss @@ -15,7 +15,7 @@ box-sizing: border-box; display: grid; grid-template-columns: 1fr 200px; - max-width: 1100px; + max-width: 80%; width: 100%; gap: var(--normal-gap-size); align-self: center; diff --git a/packages/frontend/stalker-app/src/app/modules/findings/findings-list/findings-list.component.ts b/packages/frontend/stalker-app/src/app/modules/findings/findings-list/findings-list.component.ts index 86520f8c0..9a79b008a 100644 --- a/packages/frontend/stalker-app/src/app/modules/findings/findings-list/findings-list.component.ts +++ b/packages/frontend/stalker-app/src/app/modules/findings/findings-list/findings-list.component.ts @@ -19,6 +19,15 @@ export class FindingsListComponent { this.initFindings(); } + private _filterFindingKeys: string[] = []; + public get filterFindingKeys() { + return this._filterFindingKeys; + } + + @Input() public set filterFindingKeys(value: string[]) { + this._filterFindingKeys = value; + } + public loadMoreFindings$: BehaviorSubject = new BehaviorSubject(null); public isLoadingMoreFindings$: BehaviorSubject = new BehaviorSubject(false); public findings$: Observable> | null = null; @@ -37,7 +46,7 @@ export class FindingsListComponent { this.findings$ = this.loadMoreFindings$.pipe( tap(() => this.isLoadingMoreFindings$.next(true)), scan((acc) => acc + 1, 0), - concatMap((page) => this.findingsService.getFindings(correlationKey, page, 15)), + concatMap((page) => this.findingsService.getFindings(correlationKey, page, 15, this._filterFindingKeys)), scan((acc, value) => { acc.items.push(...value.items); acc.totalRecords = value.totalRecords; diff --git a/packages/frontend/stalker-app/src/app/modules/hosts/view-host/view-host.component.scss b/packages/frontend/stalker-app/src/app/modules/hosts/view-host/view-host.component.scss index fab3192c1..659b73d6e 100644 --- a/packages/frontend/stalker-app/src/app/modules/hosts/view-host/view-host.component.scss +++ b/packages/frontend/stalker-app/src/app/modules/hosts/view-host/view-host.component.scss @@ -15,7 +15,7 @@ box-sizing: border-box; display: grid; grid-template-columns: 1fr 200px; - max-width: 1100px; + max-width: 80%; width: 100%; gap: var(--normal-gap-size); align-self: center; diff --git a/packages/frontend/stalker-app/src/app/modules/ports/view-port/view-port.component.scss b/packages/frontend/stalker-app/src/app/modules/ports/view-port/view-port.component.scss index fab3192c1..659b73d6e 100644 --- a/packages/frontend/stalker-app/src/app/modules/ports/view-port/view-port.component.scss +++ b/packages/frontend/stalker-app/src/app/modules/ports/view-port/view-port.component.scss @@ -15,7 +15,7 @@ box-sizing: border-box; display: grid; grid-template-columns: 1fr 200px; - max-width: 1100px; + max-width: 80%; width: 100%; gap: var(--normal-gap-size); align-self: center; diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html index 49e0285f4..8b4dd28f4 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html @@ -20,7 +20,132 @@

- +
+ + + Search filter + + + +
+ @for (path of sitemap$ | async; track $index) { +
+

+ {{ path }} +

+
+ } + @if (!(sitemap$ | async)?.length) { +

No available data

+ } +
+ + + @if (endpointLoading) { +
+ +
+ } + @if (endpointData$ | async; as endpointData) { + @if (!endpointLoading) { +
+ @for (field of endpointData.fields; track field) { + @if (field.type === 'text') { + @if (field.key === 'method') { +
+ +
{{ field.data }}
+
+ } + + @if (field.key === 'endpoint') { +
+ +
{{ field.data }}
+
+ } + + @if (field.key === 'statusCode') { +
+ +
+ } + } + } +
+ +
+
+ @for (field of endpointData.fields; track field) { + @if ( + field.type === 'text' && + field.key !== 'endpoint' && + field.key !== 'method' && + field.key !== 'statusCode' + ) { +
+
{{ field.label }}
+
{{ field.data }}
+
+ } + } +
+
+
+ {{ endpointData.created | timeAgo }} +
+
+
+ } + } @else { + @if (!endpointLoading) { +
+
+ search_insights +

Select an endpoint in the site map

+
+
+ } + } + + + @if (wsite.previewImage) { + Not supported yet + } @else { +
+
+ screenshot_monitor +

No image preview available

+
+
+ } + +
+ + +
@if (wsite.blocked) { diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss index 1ad22474c..149c5aed4 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss @@ -15,10 +15,167 @@ box-sizing: border-box; display: grid; grid-template-columns: 1fr 200px; - max-width: 1100px; + max-width: 80%; width: 100%; gap: var(--normal-gap-size); align-self: center; + + .content { + .website-overview { + min-height: 500px; + max-height: 600px; + display: grid; + grid-template-columns: 1fr 1fr 1fr 1fr; + grid-template-rows: 1fr 1fr 1fr; + gap: 10px; + + #sitemap { + grid-column-start: 1; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 3; + height: 100%; + overflow: hidden; + white-space: nowrap; + .sitemap-title { + margin: 5px; + font-weight: 500; + } + + .independent-scroll { + font-weight: 400; + font-size: small; + height: 100%; + overflow: scroll; + white-space: nowrap; + padding: 5px 0px 5px 5px; + + .endpoint { + margin-bottom: 2px; + margin-right: 5px; + min-width: fit-content; + cursor: pointer; + + p { + display: inline-block; + margin-right: 5px; + text-decoration: inherit; + } + } + + .endpoint:hover { + text-decoration: underline; + } + } + } + + #website-data { + grid-row-start: 1; + grid-row-end: span 1; + grid-column-start: 2; + grid-column-end: span 3; + padding: 10px; + + .spinner { + max-height: 50%; + opacity: 0.85; + } + + display: flex; + flex-direction: column; + + .inline-endpoint { + display: grid; + grid-template-columns: 1fr 10fr 1fr; + grid-template-rows: 1fr; + gap: 10px; + #method { + font-weight: 500; + grid-column-start: 1; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 1; + } + + #endpoint { + .label { + font-weight: 500; + } + grid-column-start: 2; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 1; + } + + #status-code { + grid-column-start: 3; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 1; + } + } + + .endpoint-data-divider { + margin-top: 8px; + margin-bottom: 8px; + } + + .endpoint-metadata-container { + flex-grow: 1; + display: flex; + flex-direction: column; + + .endpoint-metadata { + flex-grow: 1; + display: flex; + flex-direction: row; + gap: 10px; + width: 100%; + flex-wrap: wrap; + overflow: scroll; + + .endpoint-metadata-item { + width: 31%; + .label { + font-weight: 500; + text-decoration: underline; + } + } + } + + .timestamp-container { + width: fit-content; + .timestamp { + font-size: smaller; + opacity: 0.6; + font-style: italic; + } + } + } + } + + #preview { + grid-row-start: 2; + grid-row-end: span 2; + grid-column-start: 2; + grid-column-end: span 3; + } + } + + .section-divider { + margin-top: 10px; + margin-bottom: 10px; + } + } +} + +.no-data { + opacity: 0.65; + user-select: none; + .no-data-icon { + font-size: 4em; + fill: currentColor; + } } .show-all-button-wrapper { diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts index 556a376fe..8f36cd711 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts @@ -22,6 +22,7 @@ import { Observable, Subject, combineLatest, + debounceTime, firstValueFrom, forkJoin, map, @@ -38,6 +39,7 @@ import { ProjectSummary } from 'src/app/shared/types/project/project.summary'; import { Tag } from 'src/app/shared/types/tag.type'; import { BlockedPillTagComponent } from 'src/app/shared/widget/pill-tag/blocked-pill-tag.component'; import { TextMenuComponent } from 'src/app/shared/widget/text-menu/text-menu.component'; +import { FindingsService } from '../../../api/findings/findings.service'; import { PortsService } from '../../../api/ports/ports.service'; import { WebsitesService } from '../../../api/websites/websites.service'; import { AppHeaderComponent } from '../../../shared/components/page-header/page-header.component'; @@ -75,6 +77,7 @@ import { WebsiteInteractionsService } from '../websites-interactions.service'; MatTooltipModule, TextMenuComponent, BlockedPillTagComponent, + FindingsModule, ], selector: 'app-view-website', templateUrl: './view-website.component.html', @@ -82,6 +85,7 @@ import { WebsiteInteractionsService } from '../websites-interactions.service'; changeDetection: ChangeDetectionStrategy.OnPush, }) export class ViewWebsiteComponent implements OnDestroy { + public findingsFilterKeys = ['WebsitePathFinding']; public menuResizeObserver$?: ResizeObserver; @ViewChild('newPillTag', { read: ElementRef, static: false }) @@ -143,6 +147,31 @@ export class ViewWebsiteComponent implements OnDestroy { shareReplay(1) ); + public sitemapFilterChange$ = new BehaviorSubject(''); + public selectedEndpoint: string = ''; + public endpointLoading: boolean = false; + public selectedEndpoint$ = new Subject(); + public endpointData$ = combineLatest([this.selectedEndpoint$, this.website$]).pipe( + tap(() => { + this.endpointLoading = true; + }), + debounceTime(200), + switchMap(([endpoint, website]) => { + this.selectedEndpoint = endpoint; + return this.findingService.getLatestWebsiteEndpoint(website.correlationKey, endpoint); + }), + tap(() => { + this.endpointLoading = false; + }), + shareReplay(1) + ); + + public sitemap$ = combineLatest([this.website$, this.sitemapFilterChange$]).pipe( + map(([website, filter]) => { + return website.sitemap.filter((v) => v.toLocaleLowerCase().includes(filter.toLocaleLowerCase())); + }) + ); + public hostPorts$ = this.website$.pipe( map((website) => { const allPortSummaries = []; @@ -287,6 +316,7 @@ export class ViewWebsiteComponent implements OnDestroy { private toastr: ToastrService, private router: Router, public dialog: MatDialog, - private cdr: ChangeDetectorRef + private cdr: ChangeDetectorRef, + private findingService: FindingsService ) {} } diff --git a/packages/frontend/stalker-app/src/app/shared/shared.module.ts b/packages/frontend/stalker-app/src/app/shared/shared.module.ts index 2af0c3950..9a44dcb4a 100644 --- a/packages/frontend/stalker-app/src/app/shared/shared.module.ts +++ b/packages/frontend/stalker-app/src/app/shared/shared.module.ts @@ -36,6 +36,7 @@ import { ConfirmDialogComponent } from './widget/confirm-dialog/confirm-dialog.c import { ImageUploadComponent } from './widget/image-upload/image-upload.component'; import { NewPillTagComponent } from './widget/pill-tag/new-pill-tag.component'; import { PillTagComponent } from './widget/pill-tag/pill-tag.component'; +import { StatusCodePillTagComponent } from './widget/pill-tag/status-code-pill-tag.component'; import { SpinnerButtonComponent } from './widget/spinner-button/spinner-button.component'; import { TextMenuComponent } from './widget/text-menu/text-menu.component'; import { TextSelectMenuComponent } from './widget/text-select-menu/text-select-menu.component'; @@ -49,6 +50,7 @@ import { TextSelectMenuComponent } from './widget/text-select-menu/text-select-m WhereIdPipe, PillTagComponent, NewPillTagComponent, + StatusCodePillTagComponent, TimeAgoPipe, HumanizePipe, HumanizeDatePipe, @@ -94,6 +96,7 @@ import { TextSelectMenuComponent } from './widget/text-select-menu/text-select-m WhereIdPipe, PillTagComponent, NewPillTagComponent, + StatusCodePillTagComponent, TimeAgoPipe, HumanizePipe, HumanizeDatePipe, diff --git a/packages/frontend/stalker-app/src/app/shared/types/finding/finding.type.ts b/packages/frontend/stalker-app/src/app/shared/types/finding/finding.type.ts index 217e60332..bd6c0807b 100644 --- a/packages/frontend/stalker-app/src/app/shared/types/finding/finding.type.ts +++ b/packages/frontend/stalker-app/src/app/shared/types/finding/finding.type.ts @@ -22,4 +22,5 @@ export interface CustomFindingTextField { type: 'text'; label: string; data: string; + key: string; } diff --git a/packages/frontend/stalker-app/src/app/shared/widget/pill-tag/status-code-pill-tag.component.ts b/packages/frontend/stalker-app/src/app/shared/widget/pill-tag/status-code-pill-tag.component.ts new file mode 100644 index 000000000..718cedaf5 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/shared/widget/pill-tag/status-code-pill-tag.component.ts @@ -0,0 +1,117 @@ +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; + +@Component({ + changeDetection: ChangeDetectionStrategy.OnPush, + selector: 'app-status-code-pill-tag', + template: ` + {{ _statusCode }} + `, +}) +export class StatusCodePillTagComponent { + public readonly statusCodes = new Map([ + ['100', 'Continue'], + ['101', 'Switching Protocols'], + ['102', 'Processing (WebDAV)'], + ['103', 'Early Hints'], + ['200', 'OK'], + ['201', 'Created'], + ['202', 'Accepted'], + ['203', 'Non-Authoritative Information'], + ['204', 'No Content'], + ['205', 'Reset Content'], + ['206', 'Partial Content'], + ['207', 'Multi-Status (WebDAV)'], + ['208', 'Already Reported (WebDAV)'], + ['226', 'IM Used (HTTP Delta encoding)'], + ['300', 'Multiple Choices'], + ['301', 'Moved Permanently'], + ['302', 'Found'], + ['303', 'See Other'], + ['304', 'Not Modified'], + ['305', 'Use Proxy Deprecated'], + ['306', 'unused'], + ['307', 'Temporary Redirect'], + ['308', 'Permanent Redirect'], + ['400', 'Bad Request'], + ['401', 'Unauthorized'], + ['402', 'Payment Required Experimental'], + ['403', 'Forbidden'], + ['404', 'Not Found'], + ['405', 'Method Not Allowed'], + ['406', 'Not Acceptable'], + ['407', 'Proxy Authentication Required'], + ['408', 'Request Timeout'], + ['409', 'Conflict'], + ['410', 'Gone'], + ['411', 'Length Required'], + ['412', 'Precondition Failed'], + ['413', 'Payload Too Large'], + ['414', 'URI Too Long'], + ['415', 'Unsupported Media Type'], + ['416', 'Range Not Satisfiable'], + ['417', 'Expectation Failed'], + ['418', "I'm a teapot"], + ['421', 'Misdirected Request'], + ['422', 'Unprocessable Content (WebDAV)'], + ['423', 'Locked (WebDAV)'], + ['424', 'Failed Dependency (WebDAV)'], + ['425', 'Too Early Experimental'], + ['426', 'Upgrade Required'], + ['428', 'Precondition Required'], + ['429', 'Too Many Requests'], + ['431', 'Request Header Fields Too Large'], + ['451', 'Unavailable For Legal Reasons'], + ['500', 'Internal Server Error'], + ['501', 'Not Implemented'], + ['502', 'Bad Gateway'], + ['503', 'Service Unavailable'], + ['504', 'Gateway Timeout'], + ['505', 'HTTP Version Not Supported'], + ['506', 'Variant Also Negotiates'], + ['507', 'Insufficient Storage (WebDAV)'], + ['508', 'Loop Detected (WebDAV)'], + ['510', 'Not Extended'], + ['511', 'Network Authentication Required'], + ]); + + private readonly blue = '#0064ff'; + private readonly green = '#20ff00'; + private readonly yellow = '#cfc918'; + private readonly orange = '#c18f1f'; + private readonly red = '#a21b21'; + private readonly black = '#000000'; + + public _tagColor!: string; + public _statusCode!: string; + public _tooltip!: string; + + @Input() + public set statusCode(statusCode: string | number) { + statusCode = statusCode.toString(); + try { + const sc = Number(statusCode); + + if (sc < 100 || sc > 511) { + this._tagColor = this.black; + } else if (sc >= 100 && sc < 200) { + this._tagColor = this.black; + } else if (sc >= 200 && sc < 300) { + this._tagColor = this.green; + } else if (sc >= 300 && sc < 400) { + if (sc === 401 || sc === 403) { + this._tagColor = this.red; + } else { + this._tagColor = this.blue; + } + } else if (sc >= 400 && sc < 500) { + this._tagColor = this.yellow; + } else if (sc >= 500) { + this._tagColor = this.orange; + } + } catch { + this._tagColor = this.black; + } + this._statusCode = statusCode; + this._tooltip = this.statusCodes.get(this._statusCode) ?? ''; + } +} From 7b5f54b66f010f79c62b5d9a981904e41e45e12f Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 8 Jul 2024 10:23:11 -0400 Subject: [PATCH 08/11] adding a copy button for the URL --- .../view-website/view-website.component.html | 9 +++++++-- .../view-website/view-website.component.scss | 10 +++++++--- .../view-website/view-website.component.ts | 20 ++++++++++++++++++- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html index 8b4dd28f4..9e9b81ecd 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html @@ -67,15 +67,20 @@

-
{{ field.data }}

} @if (field.key === 'endpoint') {
-
{{ field.data }}
+ @if (!linkCopied) { + + } @else { + check + }
} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss index 149c5aed4..65871c897 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss @@ -98,13 +98,17 @@ } #endpoint { - .label { - font-weight: 500; - } grid-column-start: 2; grid-column-end: span 1; grid-row-start: 1; grid-row-end: span 1; + display: flex; + flex-direction: row; + gap: 10px; + button { + zoom: 0.75; + margin-top: -3px; + } } #status-code { diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts index 8f36cd711..cc15fe89e 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts @@ -1,3 +1,4 @@ +import { Clipboard } from '@angular/cdk/clipboard'; import { CommonModule } from '@angular/common'; import { ChangeDetectionStrategy, ChangeDetectorRef, Component, ElementRef, OnDestroy, ViewChild } from '@angular/core'; import { FormsModule } from '@angular/forms'; @@ -305,6 +306,22 @@ export class ViewWebsiteComponent implements OnDestroy { } } + public linkCopied = false; + public copyLink(path: string) { + let url = this.website.url.endsWith('/') + ? this.website.url.slice(0, this.website.url.length - 1) + : this.website.url; + + url += path.startsWith('/') ? path : '/' + path; + + this.clipboard.copy(url); + this.linkCopied = true; + setTimeout(() => { + this.linkCopied = false; + this.cdr.detectChanges(); + }, 2000); + } + constructor( private route: ActivatedRoute, private projectsService: ProjectsService, @@ -317,6 +334,7 @@ export class ViewWebsiteComponent implements OnDestroy { private router: Router, public dialog: MatDialog, private cdr: ChangeDetectorRef, - private findingService: FindingsService + private findingService: FindingsService, + private clipboard: Clipboard ) {} } From d6ba60293b8937c5dfdc63789899d0e354168eae Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 8 Jul 2024 11:40:08 -0400 Subject: [PATCH 09/11] adding tests, fixing a bug found by the tests --- .../modules/findings/findings.service.spec.ts | 199 +++++++++++++++--- .../src/modules/findings/findings.service.ts | 4 +- 2 files changed, 167 insertions(+), 36 deletions(-) diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts index 86d754667..321eed825 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts @@ -11,6 +11,8 @@ import { FindingTextField, } from '../database/reporting/findings/finding.model'; import { ProjectService } from '../database/reporting/project.service'; +import { CustomFindingFieldDto } from './finding.dto'; +import { CustomFindingsConstants } from './findings.constants'; import { FindingsService } from './findings.service'; describe('Findings Service Spec', () => { @@ -98,11 +100,7 @@ describe('Findings Service Spec', () => { it('Save - Nonexistent job - Throws', async () => { // Arrange - const c = await projectService.addProject({ - name: randomUUID(), - imageType: null, - logo: null, - }); + const c = await project(); // Act // Assert @@ -113,11 +111,7 @@ describe('Findings Service Spec', () => { it('Save - Nonexistent job - Throws', async () => { // Arrange - const c = await projectService.addProject({ - name: randomUUID(), - imageType: null, - logo: null, - }); + const c = await project(); // Act // Assert @@ -135,11 +129,7 @@ describe('Findings Service Spec', () => { 'Save - Invalid finding correlation information - Throws', async (domainName: string, ip: string, port: number) => { // Arrange - const c = await projectService.addProject({ - name: randomUUID(), - imageType: null, - logo: null, - }); + const c = await project(); const j = await jobsModel.create({ projectId: c.id, @@ -164,11 +154,7 @@ describe('Findings Service Spec', () => { it('Save - Valid findings - Returns findings', async () => { // Arrange - const c = await projectService.addProject({ - name: randomUUID(), - imageType: null, - logo: null, - }); + const c = await project(); const j = await jobsModel.create({ projectId: c.id, @@ -176,20 +162,8 @@ describe('Findings Service Spec', () => { }); // Act - await findingsService.save(c.id, j.id, { - key: 'my-finding', - name: 'My finding', - fields: [ - { - key: 'my-field', - type: 'text', - label: 'My label', - data: 'My content', - }, - ], - type: 'CustomFinding', - domainName: 'example.org', - }); + + await customDomainFinding(c.id, j.id, 'my-finding'); // Assert const correlationKey = CorrelationKeyUtils.domainCorrelationKey( @@ -212,4 +186,161 @@ describe('Findings Service Spec', () => { expect(field.label).toBe('My label'); expect(field.data).toBe('My content'); }); + + it('Get findings filtered by key', async () => { + // Arrange + const c = await project(); + + const j = await jobsModel.create({ + projectId: c.id, + task: 'CustomJob', + }); + + const filteredKey = 'my-finding-1'; + const nonFilteredKey = 'my-finding-2'; + + await customDomainFinding(c.id, j.id, filteredKey); + await customDomainFinding(c.id, j.id, nonFilteredKey); + + // Act + const correlationKey = CorrelationKeyUtils.domainCorrelationKey( + c.id, + 'example.org', + ); + const findings = await findingsService.getAll(correlationKey, 1, 100, [ + filteredKey, + ]); + + // Assert + expect(findings.totalRecords).toBe(1); + const finding = findings.items[0]; + expect(finding.key).toBe(nonFilteredKey); + }); + + it('Get latest website path finding', async () => { + // Arrange + const c = await project(); + + const j = await jobsModel.create({ + projectId: c.id, + task: 'CustomJob', + }); + + const f1 = await customWebsiteFinding( + c.id, + j.id, + CustomFindingsConstants.WebsitePathFinding, + 'example.org', + ); + const f2 = await customWebsiteFinding( + c.id, + j.id, + CustomFindingsConstants.WebsitePathFinding, + 'example.org', + ); + await customWebsiteFinding( + c.id, + j.id, + CustomFindingsConstants.WebsitePathFinding, + 'example2.com', + ); + await customWebsiteFinding( + c.id, + j.id, + CustomFindingsConstants.WebsitePathFinding, + 'example.org', + undefined, + undefined, + undefined, + undefined, + [ + { + key: CustomFindingsConstants.WebsiteEndpointFieldKey, + type: 'text', + label: 'Endpoint', + data: '/example/other-endpoint.html', + }, + ], + ); + + // Act + const correlationKey = CorrelationKeyUtils.websiteCorrelationKey( + c.id, + '1.1.1.1', + 443, + 'example.org', + '/', + ); + const endpoint = await findingsService.getLatestWebsiteEndpoint( + correlationKey, + '/example/file.html', + ); + + // Assert + expect(endpoint._id.toString()).toStrictEqual(f2._id.toString()); + expect(endpoint.fields[0].data).toStrictEqual('/example/file.html'); + }); + + async function customDomainFinding( + projectId: string, + jobId: string, + key: string, + domainName: string = 'example.org', + name: string = 'My finding', + fields: Array = [ + { + key: 'my-field', + type: 'text', + label: 'My label', + data: 'My content', + }, + ], + ) { + return await findingsService.save(projectId, jobId, { + key, + name, + fields, + type: 'CustomFinding', + domainName, + }); + } + + async function customWebsiteFinding( + projectId: string, + jobId: string, + key: string, + domainName: string = 'example.org', + ip: string = '1.1.1.1', + path: string = '/', + port: number = 443, + name: string = 'My finding', + fields: Array = [ + { + key: CustomFindingsConstants.WebsiteEndpointFieldKey, + type: 'text', + label: 'Endpoint', + data: '/example/file.html', + }, + ], + ) { + return await findingsService.save(projectId, jobId, { + key, + name, + fields, + type: 'CustomFinding', + domainName, + ip, + path, + port, + protocol: 'tcp', + }); + } + + async function project() { + return await projectService.addProject({ + name: randomUUID(), + imageType: null, + logo: null, + }); + } }); diff --git a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts index b5e077f9b..c22fdd63c 100644 --- a/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts @@ -183,7 +183,7 @@ export class FindingsService { }, }, }) - .sort({ _id: 'ascending' }); + .sort({ _id: 'descending' }); } /** @@ -231,7 +231,7 @@ export class FindingsService { key: dto.key, }; - await this.findingModel.create(finding); + return await this.findingModel.create(finding); } /** From 4c209b1ad4c24c23c8db79e0308c2895672091da Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 8 Jul 2024 11:47:53 -0400 Subject: [PATCH 10/11] fixing the removing of tags on deletion for websites --- .../service/src/modules/database/tags/tag.service.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/backend/jobs-manager/service/src/modules/database/tags/tag.service.ts b/packages/backend/jobs-manager/service/src/modules/database/tags/tag.service.ts index 56ea236e1..b48c63da3 100644 --- a/packages/backend/jobs-manager/service/src/modules/database/tags/tag.service.ts +++ b/packages/backend/jobs-manager/service/src/modules/database/tags/tag.service.ts @@ -5,6 +5,7 @@ import { Model, Types } from 'mongoose'; import { Domain } from '../reporting/domain/domain.model'; import { Host } from '../reporting/host/host.model'; import { Port } from '../reporting/port/port.model'; +import { Website } from '../reporting/websites/website.model'; import { Tag, TagsDocument } from './tag.model'; @Injectable() @@ -16,6 +17,7 @@ export class TagsService { @InjectModel('domain') private readonly domainsModel: Model, @InjectModel('host') private readonly hostsModel: Model, @InjectModel('port') private readonly portsModel: Model, + @InjectModel('websites') private readonly websiteModel: Model, ) {} public async create(text: string, color: string) { @@ -48,6 +50,10 @@ export class TagsService { .updateMany({ tags: tagId }, { $pull: { tags: tagId } }) .exec(); + this.websiteModel + .updateMany({ tags: tagId }, { $pull: { tags: tagId } }) + .exec(); + return delResult; } From 5eb643a51c51f14a775531aa22082784130421ee Mon Sep 17 00:00:00 2001 From: lm-sec Date: Mon, 8 Jul 2024 13:05:59 -0400 Subject: [PATCH 11/11] splitting the overview from the rest of the view --- .../view-website/view-website.component.html | 128 +------------- .../view-website/view-website.component.scss | 156 +---------------- .../view-website/view-website.component.ts | 44 +---- .../website-overview.component.html | 130 ++++++++++++++ .../website-overview.component.scss | 158 ++++++++++++++++++ .../website-overview.component.ts | 122 ++++++++++++++ 6 files changed, 416 insertions(+), 322 deletions(-) create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.html create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.scss create mode 100644 packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.ts diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html index 9e9b81ecd..9eb3d8130 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.html @@ -20,134 +20,8 @@

-
- - - Search filter + - - -
- @for (path of sitemap$ | async; track $index) { -
-

- {{ path }} -

-
- } - @if (!(sitemap$ | async)?.length) { -

No available data

- } -
- - - @if (endpointLoading) { -
- -
- } - @if (endpointData$ | async; as endpointData) { - @if (!endpointLoading) { -
- @for (field of endpointData.fields; track field) { - @if (field.type === 'text') { - @if (field.key === 'method') { -
-
{{ field.data }}
-
- } - - @if (field.key === 'endpoint') { -
-
{{ field.data }}
- @if (!linkCopied) { - - } @else { - check - } -
- } - - @if (field.key === 'statusCode') { -
- -
- } - } - } -
- -
-
- @for (field of endpointData.fields; track field) { - @if ( - field.type === 'text' && - field.key !== 'endpoint' && - field.key !== 'method' && - field.key !== 'statusCode' - ) { -
-
{{ field.label }}
-
{{ field.data }}
-
- } - } -
-
-
- {{ endpointData.created | timeAgo }} -
-
-
- } - } @else { - @if (!endpointLoading) { -
-
- search_insights -

Select an endpoint in the site map

-
-
- } - } - - - @if (wsite.previewImage) { - Not supported yet - } @else { -
-
- screenshot_monitor -

No image preview available

-
-
- } - -
diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss index 65871c897..d737290fa 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.scss @@ -21,151 +21,10 @@ align-self: center; .content { - .website-overview { - min-height: 500px; - max-height: 600px; - display: grid; - grid-template-columns: 1fr 1fr 1fr 1fr; - grid-template-rows: 1fr 1fr 1fr; - gap: 10px; - - #sitemap { - grid-column-start: 1; - grid-column-end: span 1; - grid-row-start: 1; - grid-row-end: span 3; - height: 100%; - overflow: hidden; - white-space: nowrap; - .sitemap-title { - margin: 5px; - font-weight: 500; - } - - .independent-scroll { - font-weight: 400; - font-size: small; - height: 100%; - overflow: scroll; - white-space: nowrap; - padding: 5px 0px 5px 5px; - - .endpoint { - margin-bottom: 2px; - margin-right: 5px; - min-width: fit-content; - cursor: pointer; - - p { - display: inline-block; - margin-right: 5px; - text-decoration: inherit; - } - } - - .endpoint:hover { - text-decoration: underline; - } - } - } - - #website-data { - grid-row-start: 1; - grid-row-end: span 1; - grid-column-start: 2; - grid-column-end: span 3; - padding: 10px; - - .spinner { - max-height: 50%; - opacity: 0.85; - } - - display: flex; - flex-direction: column; - - .inline-endpoint { - display: grid; - grid-template-columns: 1fr 10fr 1fr; - grid-template-rows: 1fr; - gap: 10px; - #method { - font-weight: 500; - grid-column-start: 1; - grid-column-end: span 1; - grid-row-start: 1; - grid-row-end: span 1; - } - - #endpoint { - grid-column-start: 2; - grid-column-end: span 1; - grid-row-start: 1; - grid-row-end: span 1; - display: flex; - flex-direction: row; - gap: 10px; - button { - zoom: 0.75; - margin-top: -3px; - } - } - - #status-code { - grid-column-start: 3; - grid-column-end: span 1; - grid-row-start: 1; - grid-row-end: span 1; - } - } - - .endpoint-data-divider { - margin-top: 8px; - margin-bottom: 8px; - } - - .endpoint-metadata-container { - flex-grow: 1; - display: flex; - flex-direction: column; - - .endpoint-metadata { - flex-grow: 1; - display: flex; - flex-direction: row; - gap: 10px; - width: 100%; - flex-wrap: wrap; - overflow: scroll; - - .endpoint-metadata-item { - width: 31%; - .label { - font-weight: 500; - text-decoration: underline; - } - } - } - - .timestamp-container { - width: fit-content; - .timestamp { - font-size: smaller; - opacity: 0.6; - font-style: italic; - } - } - } - } - - #preview { - grid-row-start: 2; - grid-row-end: span 2; - grid-column-start: 2; - grid-column-end: span 3; - } + app-website-overview { + display: block; + width: 100%; } - .section-divider { margin-top: 10px; margin-bottom: 10px; @@ -173,15 +32,6 @@ } } -.no-data { - opacity: 0.65; - user-select: none; - .no-data-icon { - font-size: 4em; - fill: currentColor; - } -} - .show-all-button-wrapper { display: flex; justify-content: center; diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts index cc15fe89e..46b999987 100644 --- a/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/view-website.component.ts @@ -23,7 +23,6 @@ import { Observable, Subject, combineLatest, - debounceTime, firstValueFrom, forkJoin, map, @@ -53,6 +52,7 @@ import { NewPillTagComponent } from '../../../shared/widget/pill-tag/new-pill-ta import { SelectItem } from '../../../shared/widget/text-select-menu/text-select-menu.component'; import { FindingsModule } from '../../findings/findings.module'; import { WebsiteInteractionsService } from '../websites-interactions.service'; +import { WebsiteOverviewComponent } from './website-overview/website-overview.component'; @Component({ standalone: true, @@ -79,6 +79,7 @@ import { WebsiteInteractionsService } from '../websites-interactions.service'; TextMenuComponent, BlockedPillTagComponent, FindingsModule, + WebsiteOverviewComponent, ], selector: 'app-view-website', templateUrl: './view-website.component.html', @@ -148,31 +149,6 @@ export class ViewWebsiteComponent implements OnDestroy { shareReplay(1) ); - public sitemapFilterChange$ = new BehaviorSubject(''); - public selectedEndpoint: string = ''; - public endpointLoading: boolean = false; - public selectedEndpoint$ = new Subject(); - public endpointData$ = combineLatest([this.selectedEndpoint$, this.website$]).pipe( - tap(() => { - this.endpointLoading = true; - }), - debounceTime(200), - switchMap(([endpoint, website]) => { - this.selectedEndpoint = endpoint; - return this.findingService.getLatestWebsiteEndpoint(website.correlationKey, endpoint); - }), - tap(() => { - this.endpointLoading = false; - }), - shareReplay(1) - ); - - public sitemap$ = combineLatest([this.website$, this.sitemapFilterChange$]).pipe( - map(([website, filter]) => { - return website.sitemap.filter((v) => v.toLocaleLowerCase().includes(filter.toLocaleLowerCase())); - }) - ); - public hostPorts$ = this.website$.pipe( map((website) => { const allPortSummaries = []; @@ -306,22 +282,6 @@ export class ViewWebsiteComponent implements OnDestroy { } } - public linkCopied = false; - public copyLink(path: string) { - let url = this.website.url.endsWith('/') - ? this.website.url.slice(0, this.website.url.length - 1) - : this.website.url; - - url += path.startsWith('/') ? path : '/' + path; - - this.clipboard.copy(url); - this.linkCopied = true; - setTimeout(() => { - this.linkCopied = false; - this.cdr.detectChanges(); - }, 2000); - } - constructor( private route: ActivatedRoute, private projectsService: ProjectsService, diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.html b/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.html new file mode 100644 index 000000000..85dc48f3a --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.html @@ -0,0 +1,130 @@ +
+
+ + + Search filter + + + +
+ @for (path of sitemap$ | async; track $index) { +
+

+ {{ path }} +

+
+ } + @if (!(sitemap$ | async)?.length) { +

No available data

+ } +
+ + + @if (endpointLoading) { +
+ +
+ } + @if (endpointData$ | async; as endpointData) { + @if (!endpointLoading) { +
+ @for (field of endpointData.fields; track field) { + @if (field.type === 'text') { + @if (field.key === 'method') { +
+
{{ field.data }}
+
+ } + + @if (field.key === 'endpoint') { +
+
{{ field.data }}
+ @if (!linkCopied) { + + } @else { + check + } +
+ } + + @if (field.key === 'statusCode') { +
+ +
+ } + } + } +
+ +
+
+ @for (field of endpointData.fields; track field) { + @if ( + field.type === 'text' && + field.key !== 'endpoint' && + field.key !== 'method' && + field.key !== 'statusCode' + ) { +
+
{{ field.label }}
+
{{ field.data }}
+
+ } + } +
+
+
+ {{ endpointData.created | timeAgo }} +
+
+
+ } + } @else { + @if (!endpointLoading) { +
+
+ search_insights +

Select an endpoint in the site map

+
+
+ } + } + + + @if (_website.previewImage) { + Not supported yet + } @else { +
+
+ screenshot_monitor +

No image preview available

+
+
+ } + +
+
diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.scss b/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.scss new file mode 100644 index 000000000..69179b630 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.scss @@ -0,0 +1,158 @@ +.content-wrapper { + box-sizing: border-box; + display: grid; + width: 100%; + gap: var(--normal-gap-size); + + .website-overview { + min-height: 500px; + max-height: 600px; + display: grid; + grid-template-columns: 1fr 1fr 1fr 1fr; + grid-template-rows: 1fr 1fr 1fr; + gap: 10px; + + #sitemap { + grid-column-start: 1; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 3; + height: 100%; + overflow: hidden; + white-space: nowrap; + + .independent-scroll { + font-weight: 400; + font-size: small; + height: 100%; + overflow: scroll; + white-space: nowrap; + padding: 5px 0px 5px 5px; + + .endpoint { + margin-bottom: 2px; + margin-right: 5px; + min-width: fit-content; + cursor: pointer; + + p { + display: inline-block; + margin-right: 5px; + text-decoration: inherit; + } + } + + .endpoint:hover { + text-decoration: underline; + } + } + } + + #website-data { + grid-row-start: 1; + grid-row-end: span 1; + grid-column-start: 2; + grid-column-end: span 3; + padding: 10px; + + .spinner { + max-height: 50%; + opacity: 0.85; + } + + display: flex; + flex-direction: column; + + .inline-endpoint { + display: grid; + grid-template-columns: 1fr 10fr 1fr; + grid-template-rows: 1fr; + gap: 10px; + + #method { + font-weight: 500; + grid-column-start: 1; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 1; + } + + #endpoint { + grid-column-start: 2; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 1; + display: flex; + flex-direction: row; + gap: 10px; + + button { + zoom: 0.75; + margin-top: -3px; + } + } + + #status-code { + grid-column-start: 3; + grid-column-end: span 1; + grid-row-start: 1; + grid-row-end: span 1; + } + } + + .endpoint-data-divider { + margin-top: 8px; + margin-bottom: 8px; + } + + .endpoint-metadata-container { + flex-grow: 1; + display: flex; + flex-direction: column; + + .endpoint-metadata { + flex-grow: 1; + display: flex; + flex-direction: row; + gap: 10px; + width: 100%; + flex-wrap: wrap; + overflow: scroll; + + .endpoint-metadata-item { + width: 31%; + .label { + font-weight: 500; + text-decoration: underline; + } + } + } + + .timestamp-container { + width: fit-content; + .timestamp { + font-size: smaller; + opacity: 0.6; + font-style: italic; + } + } + } + } + + #preview { + grid-row-start: 2; + grid-row-end: span 2; + grid-column-start: 2; + grid-column-end: span 3; + } + } +} + +.no-data { + opacity: 0.65; + user-select: none; + .no-data-icon { + font-size: 4em; + fill: currentColor; + } +} diff --git a/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.ts b/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.ts new file mode 100644 index 000000000..1e8294b57 --- /dev/null +++ b/packages/frontend/stalker-app/src/app/modules/websites/view-website/website-overview/website-overview.component.ts @@ -0,0 +1,122 @@ +import { Clipboard } from '@angular/cdk/clipboard'; +import { CommonModule } from '@angular/common'; +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, Input } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { MatCardModule } from '@angular/material/card'; +import { MatFormFieldModule } from '@angular/material/form-field'; +import { MatIconModule } from '@angular/material/icon'; +import { MatInputModule } from '@angular/material/input'; +import { MatListModule } from '@angular/material/list'; +import { MatMenuModule } from '@angular/material/menu'; +import { MatPaginatorModule } from '@angular/material/paginator'; +import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; +import { MatSidenavModule } from '@angular/material/sidenav'; +import { MatTableModule } from '@angular/material/table'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { BehaviorSubject, combineLatest, debounceTime, filter, map, shareReplay, Subject, switchMap, tap } from 'rxjs'; +import { FindingsService } from '../../../../api/findings/findings.service'; +import { SharedModule } from '../../../../shared/shared.module'; +import { Website } from '../../../../shared/types/websites/website.type'; +import { TextMenuComponent } from '../../../../shared/widget/text-menu/text-menu.component'; +import { FindingsModule } from '../../../findings/findings.module'; + +@Component({ + standalone: true, + imports: [ + CommonModule, + SharedModule, + MatCardModule, + MatIconModule, + MatListModule, + MatFormFieldModule, + MatTableModule, + MatPaginatorModule, + MatSidenavModule, + MatButtonModule, + MatInputModule, + MatProgressSpinnerModule, + MatMenuModule, + FormsModule, + FindingsModule, + MatTooltipModule, + TextMenuComponent, + FindingsModule, + ], + selector: 'app-website-overview', + templateUrl: './website-overview.component.html', + styleUrls: ['./website-overview.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class WebsiteOverviewComponent { + public websiteSubject$: BehaviorSubject = new BehaviorSubject(undefined); + public website$ = this.websiteSubject$.pipe(filter((w) => !!w)); + public _website!: Website; + + @Input() + public set website(wsite: Website) { + console.log('website setter'); + this.websiteSubject$.next(wsite); + this._website = wsite; + } + + public get website() { + return this._website; + } + + public a = this.website$.pipe( + tap((w) => { + console.log('website tap'); + console.log(w); + }) + ); + + public sitemapFilterChange$ = new BehaviorSubject(''); + public selectedEndpoint: string = ''; + public endpointLoading: boolean = false; + public selectedEndpoint$ = new Subject(); + public endpointData$ = combineLatest([this.selectedEndpoint$, this.website$]).pipe( + tap(() => { + this.endpointLoading = true; + }), + debounceTime(200), + switchMap(([endpoint, website]) => { + this.selectedEndpoint = endpoint; + return this.findingService.getLatestWebsiteEndpoint(website!.correlationKey, endpoint); + }), + tap(() => { + this.endpointLoading = false; + }), + shareReplay(1) + ); + + public sitemap$ = combineLatest([this.website$, this.sitemapFilterChange$]).pipe( + map(([website, filter]) => { + console.log('coucou!!'); + return website!.sitemap.filter((v) => v.toLocaleLowerCase().includes(filter.toLocaleLowerCase())); + }), + tap(() => this.cdr.detectChanges()) + ); + + public linkCopied = false; + public copyLink(path: string) { + let url = this.website.url.endsWith('/') + ? this.website.url.slice(0, this.website.url.length - 1) + : this.website.url; + + url += path.startsWith('/') ? path : '/' + path; + + this.clipboard.copy(url); + this.linkCopied = true; + setTimeout(() => { + this.linkCopied = false; + this.cdr.detectChanges(); + }, 2000); + } + + constructor( + private cdr: ChangeDetectorRef, + private findingService: FindingsService, + private clipboard: Clipboard + ) {} +}