Skip to content

Commit

Permalink
Introduce finding-type specific issue-replication cfg
Browse files Browse the repository at this point in the history 
Useful to toggle issue-replication per finding-type without code
changes.
  • Loading branch information
@zkdev
zkdev committed Apr 26, 2024
1 parent ffbeba1 commit 4aa54f0
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 57 deletions.
67 changes: 49 additions & 18 deletions config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import concourse.model.traits.image_scan as image_scan
import cnudie.iter
import dso.cvss
import dso.model
import gci.componentmodel as cm
import github.compliance.model
import protecode.model
Expand Down Expand Up @@ -136,6 +137,27 @@ class BDBAConfig:
delete_inactive_products_after_seconds: int


@dataclasses.dataclass(frozen=True)
class FindingTypeIssueReplicationCfgBase:
'''
:param str finding_type:
finding type this configuration should be applied for
(see cc-utils dso/model.py for available "Datatype"s)
:param bool enable_issue_assignees
'''
finding_type: str
enable_issue_assignees: bool


@dataclasses.dataclass(frozen=True)
class VulnerabilityIssueReplicationCfg(FindingTypeIssueReplicationCfgBase):
'''
:param int cve_threshold:
vulnerability findings below this threshold won't be reported in the issue(s)
'''
cve_threshold: int


@dataclasses.dataclass(frozen=True)
class IssueReplicatorConfig:
'''
Expand All @@ -145,8 +167,6 @@ class IssueReplicatorConfig:
time after which an issue must be updated at latest
:param int lookup_new_backlog_item_interval:
time to wait in case no backlog item was found before searching for new backlog item again
:param int cve_threshold:
vulnerability findings below this threshold won't be reported in the issue(s)
:param LicenseCfg license_cfg:
required to differentiate between allowed and prohibited licenses
:param MaxProcessingTimesDays max_processing_days:
Expand All @@ -159,30 +179,30 @@ class IssueReplicatorConfig:
labels matching one of these regexes won't be removed upon an issue update
:param int number_included_closed_issues:
number of closed issues to consider when evaluating creating vs re-opening an issue
:param bool enable_issue_assignees
:param tuple[str] artefact_types:
list of artefact types for which issues should be created, other artefact types are skipped
:param Callable[Node, bool] node_filter:
filter of artefact nodes to explicitly in- or exclude artefacts from the issue replication
:param tuple[RescoringRule] cve_rescoring_rules:
these rules are applied to calculate proposed rescorings which are displayed in the issue
:param tuple[FindingTypeIssueReplicationCfgBase] finding_type_issue_replication_cfgs:
these cfgs are finding type specific and allow fine granular configuration
'''
delivery_service_url: str
delivery_dashboard_url: str
replication_interval: int
lookup_new_backlog_item_interval: int
cve_threshold: int
license_cfg: image_scan.LicenseCfg
max_processing_days: github.compliance.model.MaxProcessingTimesDays
github_api_lookup: typing.Callable[[str], github3.GitHub]
github_issues_repository: github3.repos.Repository
github_issue_template_cfgs: tuple[image_scan.GithubIssueTemplateCfg]
github_issue_labels_to_preserve: set[str]
number_included_closed_issues: int
enable_issue_assignees: bool
artefact_types: tuple[str]
node_filter: typing.Callable[[cnudie.iter.Node], bool]
cve_rescoring_rules: tuple[dso.cvss.RescoringRule]
finding_type_issue_replication_cfgs: tuple[FindingTypeIssueReplicationCfgBase]


@dataclasses.dataclass(frozen=True)
Expand Down Expand Up @@ -554,11 +574,6 @@ def deserialise_issue_replicator_config(
default_value=60,
)

cve_threshold = deserialise_config_property(
config=issue_replicator_config,
property_key='cve_threshold',
)

prohibited_licenses = deserialise_config_property(
config=issue_replicator_config,
property_key='prohibited_licenses',
Expand Down Expand Up @@ -610,12 +625,6 @@ def deserialise_issue_replicator_config(
default_value=0,
)

enable_issue_assignees = deserialise_config_property(
config=issue_replicator_config,
property_key='enable_issue_assignees',
default_value=True,
)

artefact_types = tuple(deserialise_config_property(
config=issue_replicator_config,
property_key='artefact_types',
Expand Down Expand Up @@ -647,23 +656,45 @@ def deserialise_issue_replicator_config(
)
cve_rescoring_rules = tuple(dso.cvss.rescoring_rules_from_dicts(cve_rescoring_rules_raw))

finding_type_issue_replication_cfgs_raw = deserialise_config_property(
config=issue_replicator_config,
property_key='finding_type_issue_replication_configs',
default_config=default_config,
default_value=[],
)

model_class_for_finding_type = {
dso.model.Datatype.VULNERABILITY: VulnerabilityIssueReplicationCfg,
dso.model.Datatype.LICENSE: FindingTypeIssueReplicationCfgBase,
}

finding_type_issue_replication_cfgs = tuple(
dacite.from_dict(
data_class=model_class_for_finding_type.get(
finding_type_issue_replication_cfg_raw['finding_type'],
FindingTypeIssueReplicationCfgBase,
),
data=finding_type_issue_replication_cfg_raw,
)
for finding_type_issue_replication_cfg_raw in finding_type_issue_replication_cfgs_raw
)

return IssueReplicatorConfig(
delivery_service_url=delivery_service_url,
delivery_dashboard_url=delivery_dashboard_url,
replication_interval=replication_interval,
lookup_new_backlog_item_interval=lookup_new_backlog_item_interval,
cve_threshold=cve_threshold,
license_cfg=license_cfg,
max_processing_days=max_processing_days,
github_api_lookup=github_api_lookup,
github_issues_repository=github_issues_repository,
github_issue_template_cfgs=github_issue_template_cfgs,
github_issue_labels_to_preserve=github_issue_labels_to_preserve,
number_included_closed_issues=number_included_closed_issues,
enable_issue_assignees=enable_issue_assignees,
artefact_types=artefact_types,
node_filter=node_filter,
cve_rescoring_rules=cve_rescoring_rules,
finding_type_issue_replication_cfgs=finding_type_issue_replication_cfgs,
)


Expand Down
98 changes: 60 additions & 38 deletions issue_replicator/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,10 +283,10 @@ def _findings_by_type_and_date(
None,
]:
'''
yields all findings of the given `artefact` in `components`, grouped by finding type and latest
processing date. Also, it yields the information whether the artefact was scanned at all which
is determined based on if there is a "dummy finding". Thresholds provided by configuration are
applied on the findings before yielding.
yields all findings (of configured types) of the given `artefact` in `components`, grouped by
finding type and latest processing date. Also, it yields the information whether the artefact
was scanned at all which is determined based on if there is a "dummy finding". Thresholds
provided by configuration are applied on the findings before yielding.
'''
findings = tuple(_iter_findings_for_artefact(
delivery_client=delivery_client,
Expand All @@ -297,44 +297,46 @@ def _findings_by_type_and_date(

sprints = sprint_dates(delivery_client=delivery_client)

datasource_for_datatype = {
dso.model.Datatype.VULNERABILITY: dso.model.Datasource.BDBA,
dso.model.Datatype.LICENSE: dso.model.Datasource.BDBA,
dso.model.Datatype.MALWARE: dso.model.Datasource.CLAMAV,
}

for latest_processing_date in latest_processing_dates:
date = dateutil.parser.isoparse(latest_processing_date).date()

vulnerability_bdba_findings, is_scanned = _findings_for_type_and_date(
issue_replicator_config=issue_replicator_config,
latest_processing_date=date,
sprints=sprints,
type=dso.model.Datatype.VULNERABILITY,
source=dso.model.Datasource.BDBA,
findings=findings,
)
vulnerability_bdba_findings = tuple(
finding for finding in vulnerability_bdba_findings
if finding.finding.data.cvss_v3_score >= issue_replicator_config.cve_threshold
)
yield (
dso.model.Datatype.VULNERABILITY,
dso.model.Datasource.BDBA,
date,
vulnerability_bdba_findings,
is_scanned,
)
for finding_type_cfg in issue_replicator_config.finding_type_issue_replication_cfgs:
finding_type = finding_type_cfg.finding_type
finding_source = datasource_for_datatype.get(finding_type)

license_bdba_findings, is_scanned = _findings_for_type_and_date(
issue_replicator_config=issue_replicator_config,
latest_processing_date=date,
sprints=sprints,
type=dso.model.Datatype.LICENSE,
source=dso.model.Datasource.BDBA,
findings=findings,
)
yield (
dso.model.Datatype.LICENSE,
dso.model.Datasource.BDBA,
date,
license_bdba_findings,
is_scanned,
)
logger.info(f'processing findings of {finding_type=} with {finding_type_cfg=}')

findings, is_scanned = _findings_for_type_and_date(
issue_replicator_config=issue_replicator_config,
latest_processing_date=date,
sprints=sprints,
type=finding_type,
source=finding_source,
findings=findings,
)

if (
finding_type == dso.model.Datatype.VULNERABILITY
and isinstance(finding_type_cfg, config.VulnerabilityIssueReplicationCfg)
):
findings = tuple(
finding for finding in findings
if finding.finding.data.cvss_v3_score >= finding_type_cfg.cve_threshold
)

yield (
finding_type,
finding_source,
date,
findings,
is_scanned,
)


def replicate_issue(
Expand Down Expand Up @@ -413,13 +415,33 @@ def replicate_issue(
latest_processing_dates=correlation_ids_by_latest_processing_date.keys(),
)

def _find_finding_type_issue_replication_cfg(
finding_cfgs: collections.abc.Iterable[config.FindingTypeIssueReplicationCfgBase],
finding_type: str,
absent_ok: bool=False,
) -> config.FindingTypeIssueReplicationCfgBase:
for finding_cfg in finding_cfgs:
if finding_cfg.finding_type == finding_type:
return finding_cfg

if absent_ok:
return None

return ValueError(f'no finding-type specific cfg found for {finding_type=}')

is_in_bom = len(active_compliance_snapshots_for_artefact) > 0
for type, source, date, findings, is_scanned in findings_by_type_and_date:
correlation_id = correlation_ids_by_latest_processing_date.get(date.isoformat())

finding_type_issue_replication_cfg = _find_finding_type_issue_replication_cfg(
finding_cfgs=issue_replicator_config.finding_type_issue_replication_cfgs,
finding_type=type,
)

issue_replicator.github.create_or_update_or_close_issue(
cfg_name=cfg_name,
issue_replicator_config=issue_replicator_config,
finding_type_issue_replication_cfg=finding_type_issue_replication_cfg,
delivery_client=delivery_client,
type=type,
source=source,
Expand Down
3 changes: 2 additions & 1 deletion issue_replicator/github.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -679,6 +679,7 @@ def update_issue(
def create_or_update_or_close_issue(
cfg_name: str,
issue_replicator_config: config.IssueReplicatorConfig,
finding_type_issue_replication_cfg: config.FindingTypeIssueReplicationCfgBase,
delivery_client: delivery.client.DeliveryServiceClient,
type: str,
source: str,
Expand Down Expand Up @@ -763,7 +764,7 @@ def labels_to_preserve(
# not scanned yet but no open issue found either -> nothing to do
return

if issue_replicator_config.enable_issue_assignees:
if finding_type_issue_replication_cfg.enable_issue_assignees:
assignees, assignees_statuses = _issue_assignees(
issue_replicator_config=issue_replicator_config,
delivery_client=delivery_client,
Expand Down

0 comments on commit 4aa54f0

Please sign in to comment.