Support for purl (#15)

Signed-off-by: Prabhu Subramanian <[email protected]>

Dockerfile

@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/cpggen" \
       org.opencontainers.image.url="https://github.com/appthreat/cpggen" \
-      org.opencontainers.image.version="1.0.10" \
+      org.opencontainers.image.version="1.1.0" \
       org.opencontainers.image.vendor="AppThreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cpggen" \

Dockerfile-alma8

@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/cpggen" \
       org.opencontainers.image.url="https://github.com/appthreat/cpggen" \
-      org.opencontainers.image.version="1.0.10" \
+      org.opencontainers.image.version="1.1.0" \
       org.opencontainers.image.vendor="AppThreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cpggen" \

Dockerfile-oss

@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/cpggen" \
       org.opencontainers.image.url="https://github.com/appthreat/cpggen" \
-      org.opencontainers.image.version="1.0.10" \
+      org.opencontainers.image.version="1.1.0" \
       org.opencontainers.image.vendor="AppThreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cpggen" \

README.md

@@ -31,15 +31,15 @@ Download the executable binary for your operating system from the [releases page
 - cdxgen with Node.js 18 - Generates SBoM
 
 ```bash
-curl -LO https://github.com/AppThreat/cpggen/releases/download/v1.0.10/cpggen-linux-amd64
+curl -LO https://github.com/AppThreat/cpggen/releases/download/v1.1.0/cpggen-linux-amd64
 chmod +x cpggen-linux-amd64
 ./cpggen-linux-amd64 --help
 ```
 
 On Windows,
 
 ```powershell
-curl -LO https://github.com/appthreat/cpggen/releases/download/v1.0.10/cpggen.exe
+curl -LO https://github.com/appthreat/cpggen/releases/download/v1.1.0/cpggen.exe
 .\cpggen.exe --help
 ```
 
@@ -131,12 +131,16 @@ To specify input and output directory.
 cpggen -i <src directory> -o <CPG directory or file name>
 ```
 
-You can even pass a git url as source
+You can even pass a git or a package url as source
 
 ```
 cpggen -i https://github.com/HooliCorp/vulnerable-aws-koa-app -o /tmp/cpg
 ```
 
+```
+cpggen -i "pkg:maven/org.apache.commons/[email protected]" -o /tmp/cpg
+```
+
 To specify language type.
 
 ```
@@ -265,22 +269,24 @@ optional arguments:
 
 ## Environment variables
 
-| Name                    | Purpose                                                           |
-| ----------------------- | ----------------------------------------------------------------- |
-| JOERN_HOME              | Joern installation directory                                      |
-| CPGGEN_HOST             | cpggen server host. Default 127.0.0.1                             |
-| CPGGEN_PORT             | cpggen server port. Default 7072                                  |
-| CPGGEN_CONTAINER_CPU    | CPU units to use in container execution mode. Default computed    |
-| CPGGEN_CONTAINER_MEMORY | Memory units to use in container execution mode. Default computed |
-| CPGGEN_MEMORY           | Heap memory to use for frontends. Default computed                |
-| AT_DEBUG_MODE           | Set to debug to enable debug logging                              |
-| CPG_EXPORT              | Set to true to export CPG graphs in dot format                    |
-| CPG_EXPORT_REPR         | Graph to export. Default all                                      |
-| CPG_EXPORT_FORMAT       | Export format. Default dot                                        |
-| CPG_SLICE               | Set to true to slice CPG                                          |
-| CPG_SLICE_MODE          | Slice mode. Default Usages                                        |
-| SHIFTLEFT_ACCESS_TOKEN  | Set to automatically submit the CPG for analysis by Qwiet AI      |
-| CDXGEN_ARGS             | Extra arguments to pass to cdxgen                                 |
+| Name                    | Purpose                                                                    |
+| ----------------------- | -------------------------------------------------------------------------- |
+| JOERN_HOME              | Joern installation directory                                               |
+| CPGGEN_HOST             | cpggen server host. Default 127.0.0.1                                      |
+| CPGGEN_PORT             | cpggen server port. Default 7072                                           |
+| CPGGEN_CONTAINER_CPU    | CPU units to use in container execution mode. Default computed             |
+| CPGGEN_CONTAINER_MEMORY | Memory units to use in container execution mode. Default computed          |
+| CPGGEN_MEMORY           | Heap memory to use for frontends. Default computed                         |
+| AT_DEBUG_MODE           | Set to debug to enable debug logging                                       |
+| CPG_EXPORT              | Set to true to export CPG graphs in dot format                             |
+| CPG_EXPORT_REPR         | Graph to export. Default all                                               |
+| CPG_EXPORT_FORMAT       | Export format. Default dot                                                 |
+| CPG_SLICE               | Set to true to slice CPG                                                   |
+| CPG_SLICE_MODE          | Slice mode. Default Usages                                                 |
+| SHIFTLEFT_ACCESS_TOKEN  | Set to automatically submit the CPG for analysis by Qwiet AI               |
+| CDXGEN_ARGS             | Extra arguments to pass to cdxgen                                          |
+| ENABLE_SBOM             | Enable SBoM generation using cdxgen                                        |
+| JIMPLE_ANDROID_JAR      | Path to android.jar for use with jimple for .apk or .dex to CPG conversion |
 
 ## GitHub actions
 

cpggen/cli.py

@@ -135,7 +135,9 @@ def build_args():
     parser.add_argument(
         "--skip-sbom",
         action="store_true",
-        default=True if not os.getenv("SHIFTLEFT_ACCESS_TOKEN") else False,
+        default=True
+        if not os.getenv("SHIFTLEFT_ACCESS_TOKEN") and not os.getenv("ENABLE_SBOM")
+        else False,
         dest="skip_sbom",
         help="Do not generate SBoM",
     )
@@ -239,9 +241,14 @@ async def generate_cpg():
     # If src contains url, then reassign
     if not url and (src.startswith("http") or src.startswith("git")):
         url = src
-    if url.startswith("http") or url.startswith("git"):
+    if url.startswith("http") or url.startswith("git") or url.startswith("pkg:"):
         clone_dir = tempfile.mkdtemp(prefix="cpggen")
-        src = utils.clone_repo(url, clone_dir)
+        if src.startswith("pkg:"):
+            download_file = utils.download_package(src, clone_dir)
+            if download_file and os.path.exists(download_file):
+                src = clone_dir
+        else:
+            src = utils.clone_repo(url, clone_dir)
         is_temp_dir = True
     if cpg_out_dir and not os.path.exists(cpg_out_dir):
         os.makedirs(cpg_out_dir, exist_ok=True)
@@ -511,9 +518,14 @@ def main():
     if os.getenv("GITHUB_PATH") and utils.check_command("joern"):
         joern_home = ""
     is_temp_dir = False
-    if src.startswith("http") or src.startswith("git://"):
+    if src.startswith("http") or src.startswith("git://") or src.startswith("pkg:"):
         clone_dir = tempfile.mkdtemp(prefix="cpggen")
-        src = utils.clone_repo(src, clone_dir)
+        if src.startswith("pkg:"):
+            download_file = utils.download_package(src, clone_dir)
+            if download_file and os.path.exists(download_file):
+                src = clone_dir
+        else:
+            src = utils.clone_repo(src, clone_dir)
         is_temp_dir = True
         if not cpg_out_dir:
             cpg_out_dir = tempfile.mkdtemp(prefix="cpggen_cpg_out")

cpggen/executor.py

@@ -579,7 +579,9 @@ def exec_tool(
                     bin_ext=bin_ext,
                     exe_ext=exe_ext,
                     only_bat_ext=only_bat_ext,
-                    android_jar=os.getenv("JIMPLE_ANDROID_JAR", ""),
+                    android_jar=f' {os.getenv("JIMPLE_ANDROID_JAR", "").strip()}'
+                    if os.getenv("JIMPLE_ANDROID_JAR")
+                    else "",
                     os_path_sep=os.path.sep,
                     **extra_args,
                 )

cpggen/utils.py

@@ -6,6 +6,12 @@
 from pathlib import Path
 from sys import platform
 
+import httpx
+import rich.progress
+from packageurl import PackageURL
+from packageurl.contrib import purl2url
+from rich.progress import Progress
+
 GIT_AVAILABLE = False
 try:
     import git
@@ -398,10 +404,19 @@ def detect_project_type(src_dir):
     if is_exe(src_dir):
         project_types.append("binary")
     # Directory contains just a bunch of jar then try jimple
-    if not is_java_like and find_files(src_dir, ".jar", False, True):
-        if os.getenv("SHIFTLEFT_ACCESS_TOKEN"):
-            project_types.append("jar")
-        else:
+    if not is_java_like:
+        if find_files(src_dir, ".jar", False, True):
+            if os.getenv("SHIFTLEFT_ACCESS_TOKEN"):
+                project_types.append("jar")
+            else:
+                project_types.append("jimple")
+        elif (
+            find_files(src_dir, ".apk", False, True)
+            or find_files(src_dir, ".zip", False, True)
+            or find_files(src_dir, ".dex", False, True)
+            or find_files(src_dir, ".class", False, True)
+            or find_files(src_dir, ".jimple", False, True)
+        ):
             project_types.append("jimple")
     return project_types
 
@@ -411,3 +426,55 @@ def clone_repo(repo_url, clone_dir, depth=1):
         return None
     git.Repo.clone_from(repo_url, clone_dir, depth=depth)
     return clone_dir
+
+
+def build_maven_download_url(purl):
+    """
+    Return a maven download URL from the `purl` string.
+    """
+    MAVEN_CENTRAL_URL = "https://repo1.maven.org/maven2/"
+    ANDROID_MAVEN = "https://maven.google.com/"
+    url_prefix = MAVEN_CENTRAL_URL
+
+    purl_data = PackageURL.from_string(purl)
+    group = purl_data.namespace.replace(".", "/")
+    name = purl_data.name
+    version = purl_data.version
+
+    if "android" in group:
+        url_prefix = ANDROID_MAVEN
+
+    if name and version:
+        return f"{url_prefix}{group}/{name}/{version}/{name}-{version}.jar"
+
+
+def get_download_url(purl_str):
+    if purl_str.startswith("pkg:maven"):
+        return build_maven_download_url(purl_str)
+    return purl2url.get_download_url(purl_str)
+
+
+def download_package(purl_str, download_dir):
+    if not purl_str:
+        return
+    durl = get_download_url(purl_str)
+    if not durl:
+        return
+    with open(
+        os.path.join(download_dir, os.path.basename(durl)), mode="wb"
+    ) as download_file:
+        with httpx.stream("GET", durl) as response:
+            total = int(response.headers["Content-Length"])
+            with Progress(
+                "[progress.percentage]{task.percentage:>3.0f}%",
+                rich.progress.BarColumn(bar_width=None),
+                rich.progress.DownloadColumn(),
+                rich.progress.TransferSpeedColumn(),
+            ) as progress:
+                download_task = progress.add_task("Download", total=total)
+                for chunk in response.iter_bytes():
+                    download_file.write(chunk)
+                    progress.update(
+                        download_task, completed=response.num_bytes_downloaded
+                    )
+                return download_file.name