Release_Glints_Jobstreet_Kalibrr_V1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "5w_0Ej3Ps24i",
        "m1zqN--kHM2T",
        "nGw99G77PxwA"
      ],
      "authorship_tag": "ABX9TyNXI1wkd5rdZ9OkNCe3wvZ0",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ulil-azmy/FMHYedit/blob/main/Release_Glints_Jobstreet_Kalibrr_V1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kxNBoH7XeeEg"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "\n",
        "# Add debian buster\n",
        "cat > /etc/apt/sources.list.d/debian.list <<'EOF'\n",
        "deb [arch=amd64 signed-by=/usr/share/keyrings/debian-buster.gpg] http://deb.debian.org/debian buster main\n",
        "deb [arch=amd64 signed-by=/usr/share/keyrings/debian-buster-updates.gpg] http://deb.debian.org/debian buster-updates main\n",
        "deb [arch=amd64 signed-by=/usr/share/keyrings/debian-security-buster.gpg] http://deb.debian.org/debian-security buster/updates main\n",
        "EOF\n",
        "\n",
        "# Add keys\n",
        "apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DCC9EFBF77E11517\n",
        "apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 648ACFD622F3D138\n",
        "apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 112695A0E562B32A\n",
        "\n",
        "apt-key export 77E11517 | gpg --dearmour -o /usr/share/keyrings/debian-buster.gpg\n",
        "apt-key export 22F3D138 | gpg --dearmour -o /usr/share/keyrings/debian-buster-updates.gpg\n",
        "apt-key export E562B32A | gpg --dearmour -o /usr/share/keyrings/debian-security-buster.gpg\n",
        "\n",
        "# Prefer debian repo for chromium* packages only\n",
        "# Note the double-blank lines between entries\n",
        "cat > /etc/apt/preferences.d/chromium.pref << 'EOF'\n",
        "Package: *\n",
        "Pin: release a=eoan\n",
        "Pin-Priority: 500\n",
        "\n",
        "\n",
        "Package: *\n",
        "Pin: origin \"deb.debian.org\"\n",
        "Pin-Priority: 300\n",
        "\n",
        "\n",
        "Package: chromium*\n",
        "Pin: origin \"deb.debian.org\"\n",
        "Pin-Priority: 700\n",
        "EOF"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get update\n",
        "!apt-get install chromium chromium-driver"
      ],
      "metadata": {
        "id": "NAhUEKYOfMMb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install selenium"
      ],
      "metadata": {
        "id": "-kAvpRhy_SRt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from selenium import webdriver\n",
        "\n",
        "url = \"https://glints.com/id/\"\n",
        "\n",
        "options = webdriver.ChromeOptions()\n",
        "options.add_argument(\"--verbose\")\n",
        "options.add_argument('--no-sandbox')\n",
        "options.add_argument('--headless')\n",
        "options.add_argument('--disable-gpu')\n",
        "options.add_argument(\"--window-size=1920,1200\")\n",
        "options.add_argument('--disable-dev-shm-usage')\n",
        "options.add_argument(\"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\")\n",
        "\n",
        "driver = webdriver.Chrome(options=options)\n",
        "driver.get(url)\n",
        "\n",
        "# Get cookies\n",
        "cookies = driver.get_cookies()\n",
        "print(\"Obtained cookies:\")\n",
        "for cookie in cookies:\n",
        "    print(cookie)\n",
        "\n",
        "driver.quit()\n"
      ],
      "metadata": {
        "id": "SyB8S-VysWdz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Release: Glints V1**"
      ],
      "metadata": {
        "id": "COXYpBRCI8DO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "import json\n",
        "import pandas as pd\n",
        "from IPython.display import display, HTML\n",
        "import math\n",
        "from datetime import datetime, timedelta\n",
        "\n",
        "def fetch_jobs_data(offset):\n",
        "    url = \"https://glints.com/api/v2/graphql\"\n",
        "    headers = {\n",
        "      \"authority\": \"glints.com\",\n",
        "      \"accept\": \"*/*\",\n",
        "      \"accept-language\": \"id\",\n",
        "      \"cache-control\": \"no-cache\",\n",
        "      \"content-type\": \"application/json\",\n",
        "      \"cookie\": \"builderSessionId=3014259a63494eafa39f092d3b7492f9\",\n",
        "      \"origin\": \"https://glints.com\",\n",
        "      \"pragma\": \"no-cache\",\n",
        "      \"referer\": \"https://glints.com/id/opportunities/jobs/explore?keyword=%22data+analyst%22&country=ID&locationName=All+Cities%2FProvinces\",\n",
        "      \"sec-ch-ua\": '\"Not A(Brand\";v=\"99\", \"Google Chrome\";v=\"121\", \"Chromium\";v=\"121\"',\n",
        "      \"sec-ch-ua-mobile\": \"?0\",\n",
        "      \"sec-ch-ua-platform\": '\"Windows\"',\n",
        "      \"sec-fetch-dest\": \"empty\",\n",
        "      \"sec-fetch-mode\": \"cors\",\n",
        "      \"sec-fetch-site\": \"same-origin\",\n",
        "      \"traceparent\": \"00-06c7ab8d4d8ccc81ec0e4cdb7a815b74-20420b7cb0cae343-01\",\n",
        "      \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\"\n",
        "    }\n",
        "\n",
        "    data = {\n",
        "      \"operationName\": \"searchJobs\",\n",
        "        \"variables\": {\n",
        "        \"data\": {\n",
        "            \"SearchTerm\": \"\\\"data analyst\\\"\",\n",
        "            \"CountryCode\": \"ID\",\n",
        "            \"limit\": 30,\n",
        "            \"sortBy\": \"LATEST\",\n",
        "            \"offset\": 0,\n",
        "            \"includeExternalJobs\": True,\n",
        "            \"sources\": [\"NATIVE\", \"SUPER_POWERED\"],\n",
        "            \"searchVariant\": \"VARIANT_A\"\n",
        "        }\n",
        "    },\n",
        "    \"query\": \"query searchJobs($data: JobSearchConditionInput!) { searchJobs(data: $data) { jobsInPage { id title isRemote status createdAt updatedAt isActivelyHiring isHot shouldShowSalary educationLevel type fraudReportFlag salaryEstimate { minAmount maxAmount CurrencyCode } company { ...CompanyFields } citySubDivision { id name } city { ...CityFields } country { ...CountryFields } salaries { ...SalaryFields } location { ...LocationFields } minYearsOfExperience maxYearsOfExperience source type hierarchicalJobCategory { id level name children { name level id __typename } parents { id level name __typename } __typename } skills { skill { id name __typename } mustHave __typename } __typename } numberOfJobsCreatedInLast14Days totalJobs __typename } } fragment CompanyFields on Company { id name logo status IndustryId __typename } fragment CityFields on City { id name __typename } fragment CountryFields on Country { code name __typename } fragment SalaryFields on JobSalary { id salaryType salaryMode maxAmount minAmount CurrencyCode __typename } fragment LocationFields on HierarchicalLocation { id name administrativeLevelName formattedName level slug parents { id name administrativeLevelName formattedName level slug parents { level formattedName slug __typename } __typename } __typename }\"\n",
        "    }\n",
        "\n",
        "    data['variables']['data']['offset'] = offset\n",
        "\n",
        "    response = requests.post(url, headers=headers, data=json.dumps(data))\n",
        "    response_json = response.json()\n",
        "    jobs_in_page = response_json.get(\"data\", {}).get(\"searchJobs\", {}).get(\"jobsInPage\", [])\n",
        "    total_jobs = response_json.get(\"data\", {}).get(\"searchJobs\", {}).get(\"totalJobs\", 0)\n",
        "\n",
        "    return jobs_in_page, total_jobs\n",
        "\n",
        "def generate_metadata(title):\n",
        "    if not title:\n",
        "        return \"\"\n",
        "    metadata = title.lower().replace(\" & \", \"-and-\")\n",
        "    metadata = metadata.replace(\" \", \"-\").replace(\"--\", \"-\")\n",
        "    return metadata\n",
        "\n",
        "def construct_url(metadata, job_id):\n",
        "    return f\"https://glints.com/id/opportunities/jobs/{metadata}/{job_id}\"\n",
        "\n",
        "def calculate_dynamic_time_difference(updated_at):\n",
        "    # Convert the updatedAt string to a datetime object\n",
        "    updated_at_datetime = datetime.strptime(updated_at, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
        "\n",
        "    # Get the current datetime\n",
        "    current_datetime = datetime.utcnow()\n",
        "\n",
        "    # Calculate the time difference\n",
        "    time_difference = current_datetime - updated_at_datetime\n",
        "\n",
        "    # Calculate time difference in days, hours, and weeks\n",
        "    days_difference = time_difference.days\n",
        "    hours_difference = time_difference.seconds // 3600\n",
        "    weeks_difference = days_difference // 7\n",
        "\n",
        "    if weeks_difference >= 1:\n",
        "        return f\"{weeks_difference} week{'s' if weeks_difference > 1 else ''} ago\"\n",
        "    elif days_difference >= 1:\n",
        "        return f\"{days_difference} day{'s' if days_difference > 1 else ''} ago\"\n",
        "    elif hours_difference >= 1:\n",
        "        return f\"{hours_difference} hour{'s' if hours_difference > 1 else ''} ago\"\n",
        "    else:\n",
        "        return \"Less than an hour ago\"\n",
        "\n",
        "\n",
        "def generate_job_records(jobs_data):\n",
        "    job_records = []\n",
        "    for job in jobs_data:\n",
        "        company_name = job.get(\"company\", {}).get(\"name\")\n",
        "        city_name = job.get(\"city\", {}).get(\"name\")\n",
        "        status = job.get(\"company\", {}).get(\"status\")\n",
        "        updated_at = job.get(\"updatedAt\")\n",
        "        title = job.get(\"title\")\n",
        "        job_id = job.get(\"id\")\n",
        "\n",
        "        metadata = generate_metadata(title)\n",
        "        url = construct_url(metadata, job_id)\n",
        "        # Calculate dynamic time difference\n",
        "        time_difference = calculate_dynamic_time_difference(updated_at)\n",
        "\n",
        "        job_record = {\n",
        "            \"Company Name\": company_name,\n",
        "            \"City Name\": city_name,\n",
        "            \"Status\": status,\n",
        "            \"updatedAt\": updated_at,\n",
        "            \"Title\": title,\n",
        "            \"URL\": url,\n",
        "            \"Time Difference\": time_difference  # Include the dynamic time difference in a new column\n",
        "        }\n",
        "        job_records.append(job_record)\n",
        "    return job_records\n",
        "\n",
        "def display_jobs_dataframe(job_records):\n",
        "    df = pd.DataFrame(job_records)\n",
        "    df['URL'] = df['URL'].apply(lambda url: f'<a href=\"{url}\" target=\"_blank\">{url}</a>')\n",
        "\n",
        "    # Selecting only the first 5 and last 5 records\n",
        "    top_and_bottom_records = pd.concat([df.head(5), df.tail(5)])\n",
        "\n",
        "    styled_df = (\n",
        "        top_and_bottom_records.style\n",
        "        .set_table_styles([\n",
        "            {\n",
        "                'selector': 'th',\n",
        "                'props': [\n",
        "                    ('background-color', '#f5f5f5'),  # Lighter gray for header background\n",
        "                    ('color', '#333333'),  # Darker gray for text color\n",
        "                    ('font-weight', 'bold'),\n",
        "                    ('padding', '8px'),\n",
        "                    ('white-space', 'nowrap'),\n",
        "                    ('border', '1px solid #e0e0e0'),  # Thinner border for header\n",
        "                    ('font-family', 'Arial'),  # Different font family for table\n",
        "                    ('font-size', '12px')  # Smaller font size for table\n",
        "                ]\n",
        "            },\n",
        "            {\n",
        "                'selector': 'tr',\n",
        "                'props': [\n",
        "                    ('background-color', '#ffffff'),\n",
        "                    ('padding', '8px'),\n",
        "                    ('border', '1px solid #e0e0e0'),  # Thinner border for rows\n",
        "                    ('font-family', 'Arial'),\n",
        "                    ('font-size', '12px')\n",
        "                ]\n",
        "            },\n",
        "             {\n",
        "                'selector': 'td',\n",
        "                'props': [\n",
        "                    ('white-space', 'nowrap'),\n",
        "                    ('border', '1px solid #e0e0e0'),  # Thinner border for cells\n",
        "                    ('font-family', 'Arial'),\n",
        "                    ('font-size', '12px')\n",
        "                ]\n",
        "            },\n",
        "            {\n",
        "                'selector': 'tr:nth-child(even)',\n",
        "                'props': [('background-color', '#f9f9f9')]\n",
        "            },\n",
        "            {\n",
        "                'selector': 'tr:hover',\n",
        "                'props': [('background-color', '#e0e0e0')]\n",
        "            }\n",
        "        ])\n",
        "\n",
        "        .set_table_attributes('style=\"border-collapse: collapse; width: 100%;\"')  # Setting table width\n",
        "    )\n",
        "    display(HTML(styled_df.to_html(escape=False)))\n",
        "     #Save entire DataFrame as JSON\n",
        "    df.to_json('glints_job.json', orient='records')  # Save DataFrame as JSON file\n",
        "\n",
        "def main():\n",
        "    jobs_data, total_count = fetch_jobs_data(0)\n",
        "    print(f\"Total job count: {total_count}\")\n",
        "\n",
        "    total_pages = math.ceil(total_count / 30)\n",
        "    print(f\"Total pages: {total_pages}\")\n",
        "\n",
        "    job_records = []\n",
        "    for page in range(total_pages):\n",
        "        offset = page * 30\n",
        "        #print(f\"Fetching jobs with offset: {offset} and limit: 30\", end=' ')\n",
        "\n",
        "        page_jobs, _ = fetch_jobs_data(offset)\n",
        "\n",
        "        first_index = offset\n",
        "        last_index = offset + len(page_jobs) - 1\n",
        "\n",
        "        #print(f\"| First index: {first_index} | Last index: {last_index}\")\n",
        "        # Log API response details\n",
        "        #print(f\"API Response: Limit: 30, Offset: {offset}, First index: {first_index}, Last index: {last_index}\")\n",
        "        job_records.extend(generate_job_records(page_jobs))\n",
        "        # Save DataFrame as JSON\n",
        "        #df.to_json('glints_job.json', orient='records')  # Save DataFrame as JSON file\n",
        "    display_jobs_dataframe(job_records)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "id": "NjLGri_BJIpJ",
        "outputId": "aca2845b-f77b-412c-ae21-7ef50b5c2f5f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total job count: 1000\n",
            "Total pages: 34\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Release: Jobstreet V1**"
      ],
      "metadata": {
        "id": "5w_0Ej3Ps24i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "import pandas as pd\n",
        "from datetime import datetime, timedelta\n",
        "\n",
        "# Function to fetch job data from a given URL\n",
        "def fetch_job_data(url):\n",
        "    try:\n",
        "        response = requests.get(url)\n",
        "        if response.status_code == 200:\n",
        "            return response.json()\n",
        "        else:\n",
        "            print(f\"Failed to fetch data. Status code: {response.status_code}\")\n",
        "            return None\n",
        "    except requests.RequestException as e:\n",
        "        print(f\"Error occurred: {e}\")\n",
        "        return None\n",
        "\n",
        "# Function to format the listing date\n",
        "def format_date(date_string):\n",
        "    try:\n",
        "        date_object = datetime.strptime(date_string, \"%Y-%m-%dT%H:%M:%SZ\")\n",
        "        return date_object.strftime(\"%d/%m/%Y %H:%M:%S\")\n",
        "    except ValueError:\n",
        "        return 'N/A'\n",
        "\n",
        "# Function to calculate time difference from the listing date to the current date\n",
        "def calculate_time_difference(date_string):\n",
        "    try:\n",
        "        listing_date = datetime.strptime(date_string, \"%d/%m/%Y %H:%M:%S\")\n",
        "        current_date = datetime.now()\n",
        "\n",
        "        time_difference = current_date - listing_date\n",
        "\n",
        "        if time_difference < timedelta(days=1):\n",
        "            return f\"{int(time_difference.total_seconds() / 3600)} hours\"\n",
        "        elif time_difference < timedelta(days=7):\n",
        "            return f\"{int(time_difference.days)} days\"\n",
        "        else:\n",
        "            return f\"{int(time_difference.days / 7)} weeks\"\n",
        "    except ValueError:\n",
        "        return 'N/A'\n",
        "\n",
        "# Function to extract job details from each entry in the fetched data\n",
        "def extract_job_details(entry):\n",
        "    job_id = entry.get('id', 'N/A')\n",
        "    formatted_date = format_date(entry.get('listingDate', 'N/A'))\n",
        "    time_difference = calculate_time_difference(formatted_date)\n",
        "    return {\n",
        "        \"Job_Title\": entry.get('title', 'N/A'),\n",
        "        \"Company\": entry['advertiser'].get('description', 'N/A'),\n",
        "        \"Location\": entry.get('locationWhereValue', 'N/A'),\n",
        "        \"Posted_On\": formatted_date,\n",
        "        \"Time_Elapsed\": time_difference,\n",
        "        \"Salary\": entry.get('salary', 'N/A'),\n",
        "        \"Apply_Here\": f\"https://www.jobstreet.co.id/id/job/{job_id}\"\n",
        "    }\n",
        "\n",
        "# Function to fetch job data from multiple pages and extract details\n",
        "def fetch_all_job_data(base_url, params, total_pages):\n",
        "    all_data = []\n",
        "    for page in range(1, total_pages + 1):  # Iterate through all pages\n",
        "        params['page'] = str(page)\n",
        "        url = base_url + \"&\".join([f\"{key}={value}\" for key, value in params.items()])\n",
        "        print(f\"Fetching data for page {page}: {url}\")\n",
        "\n",
        "        parsed_json = fetch_job_data(url)\n",
        "        if parsed_json:\n",
        "            for entry in parsed_json.get('data', []):\n",
        "                if 'advertiser' in entry and 'description' in entry['advertiser']:\n",
        "                    job_details = extract_job_details(entry)\n",
        "                    all_data.append(job_details)\n",
        "    return all_data\n",
        "\n",
        "# Function to display a snippet of job data and save it to a JSON file\n",
        "def display_and_save_job_data(all_data):\n",
        "    df = pd.DataFrame(all_data)\n",
        "    df['Number'] = df.reset_index().index + 1  # Adding a numbering column\n",
        "    styled_df = pd.concat([df.head(10), df.tail(10)])\n",
        "    styled_df = styled_df.style.set_properties(**{'text-align': 'left'})\n",
        "    styled_df.set_table_styles([{'selector': 'th', 'props': [('text-align', 'left')]}])\n",
        "    display(styled_df)\n",
        "\n",
        "    # Save DataFrame to a single properly formatted JSON file\n",
        "    df.to_json('JobData.json', orient='records', indent=2)\n",
        "\n",
        "# Function to fetch metadata including total job count and page size\n",
        "def fetch_metadata(url):\n",
        "    try:\n",
        "        response = requests.get(url)\n",
        "        if response.status_code == 200:\n",
        "            data = response.json()\n",
        "            metadata = data.get('solMetadata', {})\n",
        "            total_job_count = metadata.get('totalJobCount', 0)\n",
        "            page_size = metadata.get('pageSize', 0)\n",
        "            return total_job_count, page_size\n",
        "        else:\n",
        "            print(f\"Failed to fetch metadata. Status code: {response.status_code}\")\n",
        "            return 0, 0\n",
        "    except requests.RequestException as e:\n",
        "        print(f\"Error occurred: {e}\")\n",
        "        return 0, 0\n",
        "\n",
        "# Function to calculate the number of pages based on total job count and page size\n",
        "def calculate_total_pages(total_jobs, page_size):\n",
        "    return (total_jobs + page_size - 1) // page_size\n",
        "\n",
        "# Function to get total job count, page size, and number of pages\n",
        "def get_job_information(base_url, params):\n",
        "    metadata_url = base_url + \"&\".join([f\"{key}={value}\" for key, value in params.items()])\n",
        "    total_jobs, page_size = fetch_metadata(metadata_url)\n",
        "    total_pages = calculate_total_pages(total_jobs, page_size)\n",
        "    return total_jobs, page_size, total_pages\n",
        "\n",
        "# Main function orchestrating the process\n",
        "def main():\n",
        "    base_url = \"https://www.jobstreet.co.id/api/chalice-search/v4/search?\"\n",
        "    params = {\n",
        "        \"siteKey\": \"ID-Main\",\n",
        "        \"keywords\": \"%22data+analyst%22\",\n",
        "        \"sortmode\": \"ListedDate\",\n",
        "        \"include\": \"seodata\",\n",
        "        \"locale\": \"id-ID\",\n",
        "    }\n",
        "\n",
        "    total_jobs, page_size, total_pages = get_job_information(base_url, params)\n",
        "\n",
        "    all_job_data = fetch_all_job_data(base_url, params, total_pages)\n",
        "    display_and_save_job_data(all_job_data)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()\n"
      ],
      "metadata": {
        "id": "fxJdKoZJs_kG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Release: Kalibrr V1**"
      ],
      "metadata": {
        "id": "m1zqN--kHM2T"
      }
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "oAbgR1t_nggm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from urllib.parse import urlencode\n",
        "import json\n",
        "homepage = \"https://www.kalibrr.id/id-ID/c/\"\n",
        "\n",
        "def log_endpoints(base_url, params):\n",
        "    all_endpoints = []\n",
        "    limit = params['limit']\n",
        "    total_count = fetch_total_count(base_url, params)\n",
        "\n",
        "    if total_count is not None:\n",
        "        for offset in range(0, total_count, limit):\n",
        "            params['offset'] = offset\n",
        "            constructed_endpoint = construct_endpoint(base_url, params)\n",
        "            all_endpoints.append(constructed_endpoint)\n",
        "\n",
        "    return all_endpoints\n",
        "\n",
        "def fetch_total_count(base_url, params):\n",
        "    try:\n",
        "        count_params = params.copy()\n",
        "        count_params['limit'] = 1\n",
        "        count_params['offset'] = 0\n",
        "\n",
        "        total_count_url = construct_endpoint(base_url, count_params)\n",
        "        #print(f\"Total Count Endpoint URL: {total_count_url}\")\n",
        "\n",
        "        response = requests.get(base_url, params=count_params)\n",
        "        response.raise_for_status()  # Raises an HTTPError for non-200 status codes\n",
        "\n",
        "        total_count = response.json().get('count', 0)\n",
        "        return total_count\n",
        "    except requests.RequestException as e:\n",
        "        print(f\"Request Exception: {e}\")\n",
        "        return None\n",
        "\n",
        "def construct_endpoint(base_url, params):\n",
        "    encoded_params = urlencode(params)\n",
        "    return f\"{base_url}?{encoded_params}\"  # Add the return statement to return the constructed URL\n",
        "\n",
        "def fetch_data_from_endpoints(endpoints):\n",
        "    job_data = []\n",
        "    number = 1  # Initialize the numbering variable\n",
        "    for endpoint in endpoints:\n",
        "        try:\n",
        "            response = requests.get(endpoint)\n",
        "            if response.status_code == 200:\n",
        "                data = response.json()\n",
        "                jobs = data.get('jobs', [])\n",
        "\n",
        "                for job in jobs:\n",
        "                    job_info = {\n",
        "                        'number': number,\n",
        "                        'job_title': job.get('name'),\n",
        "                        'company_name': job.get('company_name'),\n",
        "                        'activation_date': job.get('activation_date'),\n",
        "                        'city': job.get('google_location', {}).get('address_components', {}).get('city'),\n",
        "                        'url': f\"{homepage}{job['company']['code']}/jobs/{job['id']}/{job['slug']}\"\n",
        "                    }\n",
        "                    job_data.append(job_info)\n",
        "                    number += 1  # Increment the number for each job\n",
        "            else:\n",
        "                print(f\"Failed with status code: {response.status_code}\")\n",
        "\n",
        "        except requests.RequestException as e:\n",
        "            print(f\"Request Exception: {e}\")\n",
        "\n",
        "    return job_data\n",
        "\n",
        "\n",
        "base_url = \"https://www.kalibrr.com/kjs/job_board/search\"\n",
        "params = {\n",
        "    'limit': 15,\n",
        "    'country': 'Indonesia',\n",
        "    'sort_direction': 'asc',\n",
        "    'sort_field': 'application_end_date',\n",
        "    'text': 'Data-Analyst'\n",
        "}\n",
        "\n",
        "all_endpoints = log_endpoints(base_url, params)\n",
        "job_info = fetch_data_from_endpoints(all_endpoints)\n",
        "\n",
        "# Save the fetched job information into a JSON file\n",
        "output_file = 'job_info.json'\n",
        "with open(output_file, 'w') as file:\n",
        "    json.dump(job_info, file, indent=4)\n",
        "\n",
        "print(f\"Job information saved to '{output_file}' successfully.\")\n",
        "\n",
        "\n",
        "# Print all the constructed endpoint URLs\n",
        "#or endpoint in all_endpoints:\n",
        "    #print(endpoint)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rNdnZAorC1Jd",
        "outputId": "9e58e717-f3aa-4bc2-d408-a4fec2d4f266"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Job information saved to 'job_info.json' successfully.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **DEBUG DEBUG**"
      ],
      "metadata": {
        "id": "nGw99G77PxwA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Read the JSON file into a DataFrame\n",
        "df = pd.read_json('job_info.json')\n",
        "\n",
        "# Set display width (optional)\n",
        "pd.set_option('display.width', 1000)  # Change the value as needed\n",
        "\n",
        "# Style the DataFrame\n",
        "styled_df = pd.concat([df.head(10), df.tail(10)])\n",
        "styled_df = styled_df.style.set_properties(**{'text-align': 'left'})\n",
        "styled_df.set_table_styles([{'selector': 'th', 'props': [('text-align', 'left')]}])\n",
        "\n",
        "# Display the styled DataFrame\n",
        "display(styled_df)\n",
        "\n",
        "# Print the original DataFrame\n",
        "#print(df)\n"
      ],
      "metadata": {
        "id": "1C7aVG09P1e1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7clUf_qznj9R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **UNIFIEED IT ALL text**"
      ],
      "metadata": {
        "id": "k3MHez6inkpb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "# Configuration for each job portal\n",
        "job_portals = [\n",
        "{\n",
        "    \"name\": \"Glints\",\n",
        "    \"url\": \"https://glints.com/api/v2/graphql\",\n",
        "    \"headers\": {\n",
        "      \"authority\": \"glints.com\",\n",
        "      \"accept\": \"*/*\",\n",
        "      \"accept-language\": \"id\",\n",
        "      \"cache-control\": \"no-cache\",\n",
        "      \"content-type\": \"application/json\",\n",
        "      \"cookie\": \"builderSessionId=3014259a63494eafa39f092d3b7492f9\",\n",
        "      \"origin\": \"https://glints.com\",\n",
        "      \"pragma\": \"no-cache\",\n",
        "      \"referer\": \"https://glints.com/id/opportunities/jobs/explore?keyword=%22data+analyst%22&country=ID&locationName=All+Cities%2FProvinces\",\n",
        "      \"sec-ch-ua\": '\"Not A(Brand\";v=\"99\", \"Google Chrome\";v=\"121\", \"Chromium\";v=\"121\"',\n",
        "      \"sec-ch-ua-mobile\": \"?0\",\n",
        "      \"sec-ch-ua-platform\": '\"Windows\"',\n",
        "      \"sec-fetch-dest\": \"empty\",\n",
        "      \"sec-fetch-mode\": \"cors\",\n",
        "      \"sec-fetch-site\": \"same-origin\",\n",
        "      \"traceparent\": \"00-06c7ab8d4d8ccc81ec0e4cdb7a815b74-20420b7cb0cae343-01\",\n",
        "      \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\"\n",
        "    },\n",
        "    \"data\": {\n",
        "        \"operationName\": \"searchJobs\",\n",
        "        \"variables\": {\n",
        "            \"data\": {\n",
        "                 \"SearchTerm\": \"\\\"data analyst\\\"\",\n",
        "                  \"CountryCode\": \"ID\",\n",
        "                  \"limit\": 1,\n",
        "                  \"sortBy\": \"LATEST\",\n",
        "                  \"offset\": 0,\n",
        "                  \"includeExternalJobs\": True,\n",
        "                  \"sources\": [\"NATIVE\", \"SUPER_POWERED\"],\n",
        "                  \"searchVariant\": \"VARIANT_A\"\n",
        "            }\n",
        "        },\n",
        "       \"query\": \"query searchJobs($data: JobSearchConditionInput!) { searchJobs(data: $data) { jobsInPage { id title isRemote status createdAt updatedAt isActivelyHiring isHot shouldShowSalary educationLevel type fraudReportFlag salaryEstimate { minAmount maxAmount CurrencyCode } company { ...CompanyFields } citySubDivision { id name } city { ...CityFields } country { ...CountryFields } salaries { ...SalaryFields } location { ...LocationFields } minYearsOfExperience maxYearsOfExperience source type hierarchicalJobCategory { id level name children { name level id __typename } parents { id level name __typename } __typename } skills { skill { id name __typename } mustHave __typename } __typename } numberOfJobsCreatedInLast14Days totalJobs __typename } } fragment CompanyFields on Company { id name logo status IndustryId __typename } fragment CityFields on City { id name __typename } fragment CountryFields on Country { code name __typename } fragment SalaryFields on JobSalary { id salaryType salaryMode maxAmount minAmount CurrencyCode __typename } fragment LocationFields on HierarchicalLocation { id name administrativeLevelName formattedName level slug parents { id name administrativeLevelName formattedName level slug parents { level formattedName slug __typename } __typename } __typename }\"\n",
        "    },\n",
        "    \"structure\": {\n",
        "            \"total_jobs_key\": \"data.searchJobs.totalJobs\",\n",
        "            \"number_of_pages_key\": 1,\n",
        "            \"limit_key\": 1,\n",
        "            \"offset_key\": 0\n",
        "        }\n",
        "}\n",
        ",\n",
        "    {\n",
        "       \"name\": \"Jobstreet\",\n",
        "        \"url\": \"https://www.jobstreet.co.id/api/chalice-search/v4/search\",\n",
        "        \"params\": {\n",
        "            \"siteKey\": \"ID-Main\",\n",
        "            \"sourcesystem\": \"houston\",\n",
        "            \"userqueryid\": \"309177c185dc2dfb91aa10ca597438d9-3746302\",\n",
        "            \"userid\": \"e1073b62-7a54-4ee5-b5a9-2a3ec516d203\",\n",
        "            \"usersessionid\": \"e1073b62-7a54-4ee5-b5a9-2a3ec516d203\",\n",
        "            \"eventCaptureSessionId\": \"e1073b62-7a54-4ee5-b5a9-2a3ec516d203\",\n",
        "            \"page\": 1,\n",
        "            \"seekSelectAllPages\": True,\n",
        "            \"keywords\": \"\\\"data+analyst\\\"\",\n",
        "            \"pageSize\": 1,\n",
        "            \"include\": \"seodata\",\n",
        "            \"locale\": \"id-ID\",\n",
        "            \"solId\": \"646b632b-435c-493b-aed5-255e56db0a05\"\n",
        "            # Add more parameters if required\n",
        "        },\n",
        "        \"headers\": {\n",
        "            \"authority\": \"www.jobstreet.co.id\",\n",
        "            \"accept\": \"application/json, text/plain, */*\",\n",
        "            \"accept-language\": \"en-US,en;q=0.9,id;q=0.8\",\n",
        "            \"cache-control\": \"no-cache\",\n",
        "            \"cookie\": \"ABTestID=9538b2b8-f522-46fd-ac5f-6e77927d6dba; ABSSRP=1797; ...\",  # Truncated for brevity\n",
        "            \"pragma\": \"no-cache\",\n",
        "            \"referer\": \"https://www.jobstreet.co.id/id/jobs?keywords=%22data%2Banalyst%22\",\n",
        "            \"sec-ch-ua\": '\"Not A(Brand\";v=\"99\", \"Google Chrome\";v=\"121\", \"Chromium\";v=\"121\"',\n",
        "            \"sec-ch-ua-mobile\": \"?0\",\n",
        "            \"sec-ch-ua-platform\": '\"Windows\"',\n",
        "            \"sec-fetch-dest\": \"empty\",\n",
        "            \"sec-fetch-mode\": \"cors\",\n",
        "            \"sec-fetch-site\": \"same-origin\",\n",
        "            \"seek-request-brand\": \"jobstreet\",\n",
        "            \"seek-request-country\": \"ID\",\n",
        "            \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\n",
        "            \"x-seek-checksum\": \"129c8558\",\n",
        "            \"x-seek-site\": \"Chalice\"\n",
        "            # Add more headers if required\n",
        "        },\n",
        "       \"structure\": {\n",
        "            \"total_jobs_key\": \"totalCount\",\n",
        "            \"number_of_pages_key\": \"totalPages\",\n",
        "            \"limit_key\": 1,\n",
        "            \"offset_key\": 0\n",
        "        }\n",
        "    },\n",
        "    {\n",
        "        \"name\": \"Kalibrr\",\n",
        "        \"url\": \"https://www.kalibrr.com/kjs/job_board/search\",\n",
        "        \"params\": {\n",
        "            \"limit\": 1,\n",
        "            \"offset\": 0,\n",
        "            \"country\": \"Indonesia\",\n",
        "            \"text\": \"\\\"data+analyst\\\"\",\n",
        "            #\"sort_direction\": \"desc\",\n",
        "            #\"sort_field\": \"activation_date\"\n",
        "        },\n",
        "        \"headers\": {\n",
        "              # Not explicitly mentioned, but often present in headers\n",
        "        },\n",
        "        \"structure\": {\n",
        "            \"total_jobs_key\": \"count\",\n",
        "            \"number_of_pages_key\": 1,\n",
        "            \"limit_key\": 1,\n",
        "            \"offset_key\": 0\n",
        "        }\n",
        "    }\n",
        "    # Add more job portal configurations if needed...\n",
        "]\n",
        "\n",
        "# Function to make an API request to job portals\n",
        "def make_api_request(portal):\n",
        "    try:\n",
        "        response = requests.get(portal[\"url\"], headers=portal.get(\"headers\"), params=portal.get(\"params\"))\n",
        "        return response\n",
        "    except requests.RequestException as e:\n",
        "        print(f\"Request to {portal['name']} failed:\", str(e))\n",
        "        return None\n",
        "# Function to check connection to job portals and retrieve total job records\n",
        "def check_connection_and_get_total_jobs(job_portals):\n",
        "    for portal in job_portals:\n",
        "        response = make_api_request(portal)\n",
        "        print(f\"{portal['name']} Response Status Code:\", response.status_code if response else \"Failed\")\n",
        "\n",
        "        # Check if the response is in JSON format\n",
        "        try:\n",
        "            response_json = response.json()\n",
        "            file_name = f\"{portal['name']}_response.json\"\n",
        "            with open(file_name, 'w') as file:\n",
        "                json.dump(response_json, file, indent=4)\n",
        "                print(f\"Response for {portal['name']} saved as {file_name}\")\n",
        "\n",
        "            # Retrieve total job records if \"total_jobs_key\" is configured\n",
        "            total_jobs_key = portal.get(\"structure\", {}).get(\"total_jobs_key\")\n",
        "            if total_jobs_key and total_jobs_key in response_json:\n",
        "                total_jobs = response_json[total_jobs_key]\n",
        "                print(f\"Total jobs for {portal['name']}: {total_jobs}\")\n",
        "            else:\n",
        "                print(f\"'total_jobs_key' not found or not configured for {portal['name']}\")\n",
        "\n",
        "        except json.JSONDecodeError:\n",
        "            print(f\"Response for {portal['name']} is not in JSON format, not saving.\")\n",
        "            pass  # Handle non-JSON responses (if needed)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    check_connection_and_get_total_jobs(job_portals)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a8Rjgm69nqCL",
        "outputId": "ff3d4ad4-e5be-4c94-92a9-8f842331c5d0"
      },
      "execution_count": 134,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Glints Response Status Code: Failed\n",
            "Response for Glints saved as Glints_response.json\n",
            "'total_jobs_key' not found or not configured for Glints\n",
            "Jobstreet Response Status Code: 200\n",
            "Response for Jobstreet saved as Jobstreet_response.json\n",
            "Total jobs for Jobstreet: 126\n",
            "Kalibrr Response Status Code: 200\n",
            "Response for Kalibrr saved as Kalibrr_response.json\n",
            "Total jobs for Kalibrr: 406\n"
          ]
        }
      ]
    }
  ]
}