ground_truth.ipynb

{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "import json\n",
                "from typing import Any, Dict\n",
                "from install_test.consts import REPO_SETS\n",
                "\n",
                "\n",
                "DOCKERFILE_PATH = \"resources/dataset/dockerfiles\"\n",
                "def load_repos():\n",
                "    repos = {repo_set: json.load(open(repo_set, 'r')) for repo_set in REPO_SETS.values()}\n",
                "    return repos"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [],
            "source": [
                "def calc_visbility(repo: Dict[str, Any]):\n",
                "    # Calculate the \"visibility\" of a repo's documentation based on the following heursitic\n",
                "    # 1 / (num directories to navigate + num files to find)\n",
                "    dirs = [relevant.split('/') for relevant in repo['relevant_docs']]\n",
                "    paths = [['/'.join(repo_dirs[:i+1]) for i in range(len(repo_dirs))] for repo_dirs in dirs]\n",
                "    visibility = set([d for directories in paths for d in directories])\n",
                "    return 1 / len(visibility) if len(visibility) > 0 else 0"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "https://github.com/tiangolo/fastapi.git\n",
                        " - docs/en/docs/contributing.md, docs/en/docs/tutorial/testing.md\n",
                        " - 0.16666666666666666\n",
                        "https://github.com/psf/black.git\n",
                        " - docs/contributing/the_basics.md\n",
                        " - 0.3333333333333333\n",
                        "https://github.com/tqdm/tqdm.git\n",
                        " - \n",
                        " - 0\n",
                        "https://github.com/Textualize/rich.git\n",
                        " - CONTRIBUTING.md\n",
                        " - 1.0\n",
                        "https://github.com/OpenInterpreter/open-interpreter.git\n",
                        " - docs/CONTRIBUTING.md\n",
                        " - 0.5\n",
                        "https://github.com/home-assistant/core.git\n",
                        " - \n",
                        " - 0\n",
                        "https://github.com/sherlock-project/sherlock.git\n",
                        " - \n",
                        " - 0\n",
                        "https://github.com/explosion/spaCy.git\n",
                        " - README.md\n",
                        " - 1.0\n",
                        "https://github.com/soimort/you-get.git\n",
                        " - README.md\n",
                        " - 1.0\n",
                        "https://github.com/Textualize/textual.git\n",
                        " - CONTRIBUTING.md\n",
                        " - 1.0\n"
                    ]
                }
            ],
            "source": [
                "repos = load_repos()\n",
                "for repo in repos[REPO_SETS[-1]]:\n",
                "    print(repo['url'])\n",
                "    print(\" - \" + ', '.join(repo['relevant_docs']))\n",
                "    visibility = calc_visbility(repo)\n",
                "    repo['visibility'] = visibility\n",
                "    print(\" - \" + str(visibility))\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "from difflib import get_close_matches\n",
                "from install_test.agent.functions import _get_file_contents, get_api_url\n",
                "\n",
                "def get_dockerfile(repo, set_name):\n",
                "    name = repo['url'].split('/')[-1][:-4]\n",
                "    set_name = set_name.split('_')[-1][:-5]\n",
                "    dockerfile_path = os.path.join(DOCKERFILE_PATH, set_name, name.lower()+'.dockerfile')\n",
                "    if not os.path.exists(dockerfile_path):\n",
                "        print(f\"path {dockerfile_path} does not exist\")\n",
                "        return -2, []\n",
                "    dockerfile = open(dockerfile_path, 'r').readlines()\n",
                "\n",
                "    return dockerfile\n",
                "\n",
                "def calc_informativity(repo: Dict[str, Any], set_name: str):\n",
                "    # Calculate the \"informativity\" of a repo's documentation based on the following heuristic\n",
                "    # (Num lines in model dockerfile that appear in documentation) / (Num lines in model dockerfile total)\n",
                "\n",
                "    # step 1: load dockerfile\n",
                "    dockerfile = get_dockerfile(repo, set_name)\n",
                "\n",
                "    dockerfile = [\n",
                "        ' '.join(line.split(' ')[1:]).strip()\n",
                "        for line in dockerfile\n",
                "        if not (\n",
                "            line.startswith('COPY') or line.startswith('WORKDIR') or line.strip() == ''\n",
                "        )\n",
                "    ]\n",
                "\n",
                "    # step 2: load documents\n",
                "    if len(repo['relevant_docs']) == 0:\n",
                "        return -1, []\n",
                "    api_url = get_api_url(repo['url'])\n",
                "    documents = [_get_file_contents(api_url, path, repo['ref']).split('\\n') for path in repo['relevant_docs']]\n",
                "\n",
                "    # step 3: find dockerfile lines that occur in documentation\n",
                "    matched_lines = {}\n",
                "    for doc, doc_name in zip(documents, repo['relevant_docs']):\n",
                "        for line in doc:\n",
                "            matches = get_close_matches(line, dockerfile, cutoff=0.8)\n",
                "            if len(matches) > 0:\n",
                "                if matches[0] not in matched_lines:\n",
                "                    matched_lines[matches[0]] = []\n",
                "                matched_lines[matches[0]].append((line, doc_name))\n",
                "            else:\n",
                "                for df_line in dockerfile:\n",
                "                    if df_line in line:\n",
                "                        if df_line not in matched_lines:\n",
                "                            matched_lines[df_line] = []\n",
                "                        matched_lines[df_line].append((line, doc_name))\n",
                "\n",
                "    return (len(matched_lines) / len(dockerfile), matched_lines)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "https://github.com/mandarons/icloud-drive-docker.git 8cbcc2c\n",
                        " - \n",
                        "https://github.com/typeddjango/django-stubs.git b325955\n",
                        " - CONTRIBUTING.md\n",
                        "https://github.com/PennyLaneAI/pennylane.git b78565c\n",
                        " - .github/CONTRIBUTING.md, doc/development/guide/installation.rst, doc/development/guide/tests.rst\n",
                        "https://github.com/CVHub520/X-AnyLabeling.git 7c62ffb\n",
                        " - docs/en/get_started.md\n",
                        "https://github.com/open-compass/opencompass.git 7c7fa36\n",
                        " - README.md, docs/en/get_started/installation.md\n",
                        "https://github.com/SciPhi-AI/R2R.git 712defb\n",
                        " - docs/documentation/installation/local-system.mdx\n",
                        "https://github.com/VainF/Torch-Pruning.git 88e7472\n",
                        " - README.md\n",
                        "https://github.com/scverse/scvi-tools.git e8f9b36\n",
                        " - docs/developer/code.md\n",
                        " - informativity = 0.3333333333333333\n",
                        " - visibility = 0.3333333333333333\n",
                        "https://github.com/sabnzbd/sabnzbd.git 07250aa\n",
                        " - README.md\n",
                        "https://github.com/dlt-hub/dlt.git 6451bd7\n",
                        " - CONTRIBUTING.md\n",
                        "https://github.com/camel-ai/camel.git b8e1f5c\n",
                        " - README.md, CONTRIBUTING.md, docs/get_started/setup.md\n",
                        "https://github.com/boto/boto3.git 4e31351\n",
                        " - README.rst\n",
                        "https://github.com/cloud-custodian/cloud-custodian.git 59a1944\n",
                        " - docs/source/developer/installing.rst\n",
                        "https://github.com/aimhubio/aim.git a566d4a\n",
                        " - CONTRIBUTING.md\n",
                        "https://github.com/speechbrain/speechbrain.git dd65e12\n",
                        " - README.md, docs/contributing.md\n",
                        "https://github.com/nonebot/nonebot2.git be4f360\n",
                        " - CONTRIBUTING.md\n",
                        "https://github.com/getmoto/moto.git 255d2b8\n",
                        " - CONTRIBUTING.md, docs/docs/contributing/installation.rst, docs/docs/getting_started.rst\n",
                        "https://github.com/jxnl/instructor.git 959097e\n",
                        " - \n",
                        "https://github.com/numba/numba.git a344e8f\n",
                        " - docs/source/user/installing.rst, docs/source/developer/contributing.rst\n",
                        "https://github.com/pymc-devs/pymc.git 0cc291a\n",
                        " - docs/source/contributing/running_the_test_suite.md, docs/source/contributing/pr_tutorial.md\n",
                        "https://github.com/ranaroussi/yfinance.git 3fe87cb\n",
                        " - \n",
                        "https://github.com/beetbox/beets.git 54b2435\n",
                        " - CONTRIBUTING.rst\n",
                        "https://github.com/encode/starlette.git 2d0dde8\n",
                        " - docs/contributing.md, scripts/README.md\n",
                        "https://github.com/huggingface/datasets.git ca58154\n",
                        " - CONTRIBUTING.md\n",
                        "https://github.com/python/mypy.git 0c10367\n",
                        " - CONTRIBUTING.md\n",
                        "https://github.com/sympy/sympy.git 05e3ffc\n",
                        " - README.md, doc/src/install.md\n",
                        "https://github.com/ydataai/ydata-profiling.git e94e500\n",
                        " - docs/support-contribution/contribution_guidelines.md, README.md\n",
                        "https://github.com/spotDL/spotify-downloader.git 866dadd\n",
                        " - docs/CONTRIBUTING.md, README.md\n",
                        "https://github.com/microsoft/qlib.git ca9f186\n",
                        " - docs/developer/code_standard_and_dev_guide.rst, README.md\n",
                        "https://github.com/secdev/scapy.git 30b0398\n",
                        " - README.md, doc/scapy/development.rst\n",
                        "https://github.com/tiangolo/fastapi.git 212fd5e\n",
                        " - docs/en/docs/contributing.md, docs/en/docs/tutorial/testing.md\n",
                        "https://github.com/psf/black.git 98a580b\n",
                        " - docs/contributing/the_basics.md\n",
                        "https://github.com/tqdm/tqdm.git 951a2ba\n",
                        " - \n",
                        "https://github.com/Textualize/rich.git d0de442\n",
                        " - CONTRIBUTING.md\n",
                        "https://github.com/OpenInterpreter/open-interpreter.git dbc5259\n",
                        " - docs/CONTRIBUTING.md\n",
                        "https://github.com/home-assistant/core.git 7555f20\n",
                        " - \n",
                        "https://github.com/sherlock-project/sherlock.git f7075e1\n",
                        " - \n",
                        "https://github.com/explosion/spaCy.git 30f1f33\n",
                        " - README.md\n",
                        "https://github.com/soimort/you-get.git c4042d0\n",
                        " - README.md\n",
                        "https://github.com/Textualize/textual.git 4c0efd5\n",
                        " - CONTRIBUTING.md\n"
                    ]
                }
            ],
            "source": [
                "# CALCULATE BOTH HEURISTICS FOR ALL REPOS\n",
                "for repo_set in REPO_SETS:\n",
                "    for repo in repos[repo_set]:\n",
                "        print(repo['url'], repo['ref'])\n",
                "        print(\" - \" + ', '.join(repo['relevant_docs']))\n",
                "        if 'info' not in repo or repo['info'] is None:\n",
                "            info = calc_informativity(repo, repo_set)\n",
                "            repo['info'] = info[0]\n",
                "            print(\" - informativity = \" + str(info[0]))\n",
                "        if 'visibility' not in repo or repo['visibility'] is None:\n",
                "            visibility = calc_visbility(repo)\n",
                "            repo['visibility'] = visibility\n",
                "            print(\" - visibility = \" + str(visibility))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "metadata": {},
            "outputs": [],
            "source": [
                "# WRITE HEURISTICS  \n",
                "for repo_set in REPO_SETS:\n",
                "    with open(repo_set, 'w') as f:\n",
                "        json.dump(repos[repo_set], f)"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": ".venv",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.12"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}