Copy_of_leash_notebook_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "mount_file_id": "1ziDT7S8FyFR55AXBEYMvpu1K_cAsiv8J",
      "authorship_tag": "ABX9TyNOhpMD21g0k1UWsPIUnvXt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/christophergaughan/ChristopherGaughan.io/blob/master/Copy_of_leash_notebook_1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iwmhFZz_lNp8"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/test.csv')"
      ],
      "metadata": {
        "id": "A_M9KSdrlcb7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.head()"
      ],
      "metadata": {
        "id": "Vvj3hAvWls31"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.describe()"
      ],
      "metadata": {
        "id": "E6lkaA6PlxMk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install dask\n"
      ],
      "metadata": {
        "id": "P3i4bsE9n82q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import dask.dataframe as dd\n"
      ],
      "metadata": {
        "id": "_qfvxay0oddQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "YX03l7cPoj9T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_dd = dd.read_csv('/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/test.csv')\n"
      ],
      "metadata": {
        "id": "wwwbgKXBowoh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_dd.head()"
      ],
      "metadata": {
        "id": "8-JFKS1bpBrl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nan_counts = data_dd.isnull().sum().compute()\n"
      ],
      "metadata": {
        "id": "xGLd4OQ8pHLq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pyarrow\n"
      ],
      "metadata": {
        "id": "LzQjd3tDpesX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "metadata": {
        "id": "sWvpdLID8LWF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Replace 'path_to_your_file' with the actual path to your Parquet file\n",
        "file_path = '/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/train.parquet'\n",
        "\n",
        "# Load the Parquet file\n",
        "df_train_parq = pd.read_parquet(file_path)\n"
      ],
      "metadata": {
        "id": "jxTZAH--8fTF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install duckdb"
      ],
      "metadata": {
        "id": "69fEX8FnEQu_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import dask.dataframe as dd\n"
      ],
      "metadata": {
        "id": "a42xwnvYFH3g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_train_parquet_dask = dd.read_parquet('/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/train.parquet')\n"
      ],
      "metadata": {
        "id": "dd41Htt8cuDW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import pyarrow as pa\n",
        "import pyarrow.parquet as pq\n",
        "from tqdm.auto import tqdm\n",
        "from matplotlib import pyplot"
      ],
      "metadata": {
        "id": "qqDgSzucESrQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df_train_parquet_dask.head())\n"
      ],
      "metadata": {
        "id": "Z1n4DpaxFEK3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Sample a fraction of the data to ensure it fits into memory\n",
        "sample_df = df_train_parquet_dask['binds'].sample(frac=0.01).compute()\n",
        "\n",
        "# Now sample_df is a pandas Series and can be plotted directly\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "sns.histplot(sample_df)\n",
        "plt.show()\n",
        "\n"
      ],
      "metadata": {
        "id": "P8-nHHRZdA7U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## let's look at the proportion of molecules that actually bind"
      ],
      "metadata": {
        "id": "Ot8eSi8Yf9en"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Filter the DataFrame to include only rows where 'binds' is 1\n",
        "binding_molecules = df_train_parquet_dask[df_train_parquet_dask['binds'] == 1]\n",
        "\n",
        "# You can now perform operations on this filtered DataFrame.\n",
        "# For example, to see the first few entries:\n",
        "print(binding_molecules.head())\n"
      ],
      "metadata": {
        "id": "YuHg2JkrdakK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Filter the DataFrame to include only rows where 'binds' is 1\n",
        "binding_molecules = df_train_parquet_dask[df_train_parquet_dask['binds'] > 0]\n",
        "\n",
        "# You can now perform operations on this filtered DataFrame.\n",
        "# For example, to see the first few entries:\n",
        "print(binding_molecules.head())\n"
      ],
      "metadata": {
        "id": "bqHVpWkngV_L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Be cautious with this operation; ensure the resulting pandas DataFrame will fit into your available memory\n",
        "binding_molecules_df = binding_molecules.compute()\n",
        "\n",
        "# Now you can use binding_molecules_df as a regular pandas DataFrame\n"
      ],
      "metadata": {
        "id": "jfOzAQLygdfy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "binding_molecules_df"
      ],
      "metadata": {
        "id": "Zn0BGc1yg7Ts"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Working with duck.bb"
      ],
      "metadata": {
        "id": "I6OcsQkUqvAh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# import duckdb\n",
        "\n",
        "# # Assuming you have a Dask DataFrame 'df_train_parquet_dask'\n",
        "# # First, you'll want to convert your Dask DataFrame to a Pandas DataFrame\n",
        "# # Ensure this DataFrame isn't too large for your memory\n",
        "# binding_molecules_df = df_train_parquet_dask[df_train_parquet_dask['binds'] == 1].compute()\n",
        "\n",
        "# # Convert the Pandas DataFrame to a `duckdb` query result\n",
        "# query_result = duckdb.query_df(binding_molecules_df, \"binding_molecules_df\", \"SELECT * FROM binding_molecules_df\")\n",
        "\n",
        "# # Save the query result to a markdown file\n",
        "# query_result.markdown('/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/molecules_that_work.md')\n"
      ],
      "metadata": {
        "id": "82UHkQNXkOn_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import duckdb\n",
        "\n",
        "# Initialize a DuckDB connection\n",
        "con = duckdb.connect(database=':memory:', read_only=False)\n"
      ],
      "metadata": {
        "id": "piKKG62dmHZ3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Assuming 'train_path' is the file path to your train.parquet file\n",
        "train_path = '/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/train.parquet'\n",
        "\n",
        "# Perform an SQL query to get the count of molecules that bind versus those that don't\n",
        "binds_stats = con.execute(f\"\"\"\n",
        "    SELECT binds, count(*) as binds_count\n",
        "    FROM parquet_scan('{train_path}')\n",
        "    GROUP BY binds\n",
        "    ORDER BY count(*) DESC\n",
        "\"\"\").fetchdf()\n",
        "\n",
        "print(binds_stats)\n"
      ],
      "metadata": {
        "id": "yGRGfB7Gq3Uo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tabulate"
      ],
      "metadata": {
        "id": "91r9VgS-tOH8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import duckdb\n",
        "import pandas as pd\n",
        "\n",
        "# Ensure you have tabulate installed which is required for to_markdown\n",
        "# !pip install tabulate\n",
        "\n",
        "# Initialize DuckDB connection\n",
        "con = duckdb.connect(database=':memory:', read_only=False)\n",
        "\n",
        "# Set the path to your Parquet file\n",
        "train_path = '/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/train.parquet'\n",
        "\n",
        "# Execute the query to select rows where 'binds' is 1\n",
        "binding_query = f\"\"\"\n",
        "SELECT *\n",
        "FROM parquet_scan('{train_path}')\n",
        "WHERE binds = 1\n",
        "\"\"\"\n",
        "\n",
        "# Fetch the query results into a Pandas DataFrame\n",
        "binding_molecules_df = con.execute(binding_query).fetchdf()\n",
        "\n",
        "# Convert the Pandas DataFrame to Markdown\n",
        "markdown_str = binding_molecules_df.to_markdown()\n",
        "\n",
        "# Specify the file path for the markdown file\n",
        "markdown_file_path = '/content/drive/MyDrive/leash-BELKA.zip (Unzipped Files)/molecules_that_bind.md'\n",
        "\n",
        "# Save the markdown string to a file\n",
        "with open(markdown_file_path, 'w') as f:\n",
        "    f.write(markdown_str)\n",
        "\n",
        "# Close the DuckDB connection\n",
        "con.close()\n"
      ],
      "metadata": {
        "id": "MFZrIwtrtL_b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import duckdb\n",
        "import pandas as pd\n",
        "\n",
        "# Ensure you have tabulate installed which is required for to_markdown\n",
        "\n",
        "\n",
        "# Initialize DuckDB connection\n",
        "con = duckdb.connect(database=':memory:', read_only=False)\n",
        "\n",
        "# Set the path to your Parquet file\n",
        "train_path = '/content/drive/My Drive/path_to_your_file/train.parquet'\n",
        "\n",
        "# Execute the query to select rows where 'binds' is 1\n",
        "binding_query = f\"\"\"\n",
        "SELECT *\n",
        "FROM parquet_scan('{train_path}')\n",
        "WHERE binds = 1\n",
        "\"\"\"\n",
        "\n",
        "# Fetch the query results into a Pandas DataFrame\n",
        "binding_molecules_df = con.execute(binding_query).fetchdf()\n",
        "\n",
        "# Convert the Pandas DataFrame to Markdown\n",
        "markdown_str = binding_molecules_df.to_markdown()\n",
        "\n",
        "# Specify the file path for the markdown file\n",
        "markdown_file_path = '/content/drive/My Drive/molecules_that_bind.md'\n",
        "\n",
        "# Save the markdown string to a file\n",
        "with open(markdown_file_path, 'w') as f:\n",
        "    f.write(markdown_str)\n",
        "\n",
        "# Close the DuckDB connection\n",
        "con.close()\n"
      ],
      "metadata": {
        "id": "8ygLpiuArojB"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}