Scorecard Integration

scanpipe/migrations/0070_alter_project_purl_discoveredpackagescore_and_more.py

@@ -0,0 +1,49 @@
+# Generated by Django 5.1.3 on 2024-12-02 22:53
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0069_project_purl'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='purl',
+            field=models.CharField(blank=True, help_text="Package URL (PURL) for the project, required for pushing the project's scan result to FederatedCode. For example, if the input is an input URL like https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz, the corresponding PURL would be pkg:npm/[email protected].", max_length=2048),
+        ),
+        migrations.CreateModel(
+            name='DiscoveredPackageScore',
+            fields=[
+                ('scoring_tool', models.CharField(blank=True, choices=[('ossf-scorecard', 'Ossf'), ('others', 'Others')], help_text='Defines the source of a score or any other scoring metricsFor example: ossf-scorecard for scorecard data', max_length=100)),
+                ('scoring_tool_version', models.CharField(blank=True, help_text='Defines the version of the scoring tool used for scanning thepackageFor Eg : 4.6 current version of OSSF - scorecard', max_length=50)),
+                ('score', models.CharField(blank=True, help_text='Score of the package which is scanned', max_length=50)),
+                ('scoring_tool_documentation_url', models.CharField(blank=True, help_text='Documentation URL of the scoring tool used', max_length=100)),
+                ('score_date', models.DateTimeField(blank=True, editable=False, help_text='Date when the scoring was calculated on the package', null=True)),
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')),
+                ('discovered_package', models.ForeignKey(blank=True, editable=False, help_text='The package for which the score is given', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='discovered_packages_score', to='scanpipe.discoveredpackage')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ScorecardCheck',
+            fields=[
+                ('check_name', models.CharField(blank=True, help_text='Defines the name of check corresponding to the OSSF scoreFor example: Code-Review or CII-Best-PracticesThese are the some of the checks which are performed on a scanned package', max_length=100)),
+                ('check_score', models.CharField(blank=True, help_text='Defines the score of the check for the package scannedFor Eg : 9 is a score given for Code-Review', max_length=50)),
+                ('reason', models.CharField(blank=True, help_text='Gives a reason why a score was given for a specific checkFor eg, : Found 9/10 approved changesets -- score normalized to 9', max_length=300)),
+                ('details', models.JSONField(blank=True, default=list, help_text='A list of details/errors regarding the score')),
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')),
+                ('for_package_score', models.ForeignKey(blank=True, editable=False, help_text='The checks for which the score is given', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='discovered_packages_score_checks', to='scanpipe.discoveredpackagescore')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

scanpipe/models.py

@@ -29,6 +29,7 @@
 from collections import Counter
 from collections import defaultdict
 from contextlib import suppress
+from datetime import datetime
 from itertools import groupby
 from operator import itemgetter
 from pathlib import Path
@@ -86,6 +87,8 @@
 from rq.exceptions import NoSuchJobError
 from rq.job import Job
 from rq.job import JobStatus
+from scorecode.contrib.django.models import PackageScoreMixin
+from scorecode.contrib.django.models import ScorecardChecksMixin
 from taggit.managers import TaggableManager
 from taggit.models import GenericUUIDTaggedItemBase
 from taggit.models import TaggedItemBase
@@ -3964,6 +3967,103 @@ def as_spdx(self):
         )
 
 
+class DiscoveredPackageScore(UUIDPKModel, PackageScoreMixin):
+    def __str__(self):
+        return self.score or str(self.uuid)
+
+    discovered_package = models.ForeignKey(
+        DiscoveredPackage,
+        related_name="discovered_packages_score",
+        help_text=_("The package for which the score is given"),
+        on_delete=models.CASCADE,
+        editable=False,
+        blank=True,
+        null=True,
+    )
+
+    def parse_score_date(date_str, formats=None):
+        """
+        Parse a date string into a timezone-aware datetime object,
+        or return None if parsing fails.
+        """
+        if not formats:
+            formats = ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%SZ"]
+
+        if date_str:
+            for fmt in formats:
+                try:
+                    naive_datetime = datetime.strptime(date_str, fmt)
+                    return timezone.make_aware(
+                        naive_datetime, timezone.get_current_timezone()
+                    )
+                except ValueError:
+                    continue
+
+        # Return None if date_str is None or parsing fails
+        return None
+
+    @classmethod
+    @transaction.atomic()
+    def create_from_scorecard_data(
+        cls, discovered_package, scorecard_data, scoring_tool=None
+    ):
+        """Create ScoreCard object from scorecard data and discovered package"""
+        final_data = {
+            "score": scorecard_data.score,
+            "scoring_tool_version": scorecard_data.scoring_tool_version,
+            "scoring_tool_documentation_url": (
+                scorecard_data.scoring_tool_documentation_url
+            ),
+            "score_date": cls.parse_score_date(scorecard_data.score_date),
+        }
+
+        scorecard_object = cls.objects.create(
+            **final_data,
+            discovered_package=discovered_package,
+            scoring_tool=scoring_tool,
+        )
+
+        for check in scorecard_data.checks:
+            ScorecardCheck.create_from_data(package_score=scorecard_object, check=check)
+
+        return scorecard_object
+
+    @classmethod
+    def create_from_package_and_scorecard(cls, scorecard_data, package):
+        score_object = cls.create_from_scorecard_data(
+            discovered_package=package,
+            scorecard_data=scorecard_data,
+            scoring_tool="ossf-scorecard",
+        )
+        return score_object
+
+
+class ScorecardCheck(UUIDPKModel, ScorecardChecksMixin):
+    def __str__(self):
+        return self.check_score or str(self.uuid)
+
+    for_package_score = models.ForeignKey(
+        DiscoveredPackageScore,
+        related_name="discovered_packages_score_checks",
+        help_text=_("The checks for which the score is given"),
+        on_delete=models.CASCADE,
+        editable=False,
+        blank=True,
+        null=True,
+    )
+
+    @classmethod
+    def create_from_data(cls, package_score, check):
+        """Create a ScorecardCheck instance from provided data."""
+        return cls.objects.create(
+            check_name=check.check_name,
+            check_score=check.check_score,
+            reason=check.reason or "",
+            details=check.details or [],
+            for_package_score=package_score,
+        )
+
+
 def normalize_package_url_data(purl_mapping, ignore_nulls=False):
     """
     Normalize a mapping of purl data so database queries with

scanpipe/pipelines/fetch_scorecode_info.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+
+from scorecode import ossf_scorecard
+
+from scanpipe.models import DiscoveredPackageScore
+from scanpipe.pipelines import Pipeline
+
+
+class FetchScoreCodeInfo(Pipeline):
+    """
+    Fetch ScoreCode information for packages and dependencies.
+
+    This pipeline retrieves ScoreCode data for each package in the project
+    and stores it in the corresponding package instances.
+    """
+
+    download_inputs = False
+    is_addon = True
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_scorecode_service_availability,
+            cls.fetch_packages_scorecode_info,
+        )
+
+    def check_scorecode_service_availability(self):
+        """Check if the scorecode service is configured and available."""
+        if not ossf_scorecard.is_available():
+            raise Exception("scorecode service is not available.")
+
+    def fetch_packages_scorecode_info(self):
+        """Fetch scorecode information for each of the project's discovered packages."""
+        for package in self.project.discoveredpackages.all():
+            scorecard_data = ossf_scorecard.fetch_scorecard_info(
+                package=package, logger=None
+            )
+
+            if scorecard_data:
+                DiscoveredPackageScore.create_from_package_and_scorecard(
+                    scorecard_data=scorecard_data,
+                    package=package,
+                )
+
+            else:
+                # We Want to create error instead of exception
+                raise Exception("No data found for the package")

scanpipe/tests/__init__.py

@@ -20,9 +20,11 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
+import json
 import os
 import uuid
 from datetime import datetime
+from pathlib import Path
 from unittest import mock
 
 from django.apps import apps
@@ -284,3 +286,10 @@ def make_dependency(project, **extra):
         "license_key": "mpl-2.0",
     },
 }
+
+scorecard_data = None
+
+data = Path(__file__).parent / "data"
+
+with open(f"{data}/scorecode/scorecard_response.json") as file:
+    scorecard_data = json.load(file)