job ads.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "\n",
    "# path to the folder containing csv files\n",
    "path = \"C:\\\\Users\\\\rafae\\\\OneDrive\\\\Área de Trabalho\\\\data\"\n",
    "\n",
    "# use glob to get a list of all csv files in the folder\n",
    "all_files = glob.glob(path + \"/*.csv\")\n",
    "\n",
    "# create an empty list to store the dataframes for each file\n",
    "dfs = []\n",
    "\n",
    "# loop through each file and read it using pandas read_csv\n",
    "for filename in all_files:\n",
    "    df = pd.read_csv(filename)\n",
    "    dfs.append(df)\n",
    "\n",
    "# concatenate all dataframes into one\n",
    "all_data = pd.concat(dfs, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.to_csv(\"data_analyst.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14636"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#all_data.to_csv(\"data_analyst.csv\",index=False)\n",
    "all_data.shape[0]\n",
    "all_data.drop_duplicates(subset='Job_ID', keep='first')\n",
    "all_data.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       NaN\n",
       "1       NaN\n",
       "2       NaN\n",
       "3       NaN\n",
       "4       NaN\n",
       "         ..\n",
       "14631   NaN\n",
       "14632   NaN\n",
       "14633   NaN\n",
       "14634   NaN\n",
       "14635   NaN\n",
       "Name: Description, Length: 14636, dtype: float64"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#all_data[all_data['Description'].str.contains('salary')]\n",
    "#Competitive salary\n",
    "import re\n",
    "regex_pattern = r'\\$([\\d,]+)\\s*(?:-\\s*\\$([\\d,]+))?'#r'\\$([\\d,]+)\\s*(?:-|\\bto\\b)\\s*\\$([\\d,]+)|\\$([\\d,]+)'\n",
    "#\n",
    "def extract_salary(description):\n",
    "    match = re.search(regex_pattern, description)\n",
    "    if match:\n",
    "        salary_range = match.groups()\n",
    "        if salary_range[1]: # salary is a range\n",
    "            salary_min = float(salary_range[0].replace(',', ''))\n",
    "            salary_max = float(salary_range[1].replace(',', ''))\n",
    "            return (salary_min + salary_max) / 2\n",
    "        else: # salary is a single value\n",
    "            salary = float(salary_range[0].replace(',', ''))\n",
    "            return salary\n",
    "    else: # no salary information found in description\n",
    "        return None\n",
    "\n",
    "# apply extract_salary function to Description column\n",
    "all_data['Description'].apply(extract_salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_salary(description):\n",
    "    match = re.search(regex_pattern, description)\n",
    "    if match:\n",
    "        salary = match.group(1).replace(',', '')\n",
    "        return float(salary)\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "all_data['Salary'] = all_data['Description'].apply(extract_salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Place\n",
       "Berlin, Berlin, Germany                 863\n",
       "Seattle, WA                             826\n",
       "London, England, United Kingdom         807\n",
       "Barcelona, Catalonia, Spain             727\n",
       "Boston, MA                              711\n",
       "Toronto, Ontario, Canada                710\n",
       "Vancouver, British Columbia, Canada     663\n",
       "Madrid, Community of Madrid, Spain      551\n",
       "London, England, United Kingdom         441\n",
       "Seattle, WA                             274\n",
       "Redmond, WA                             258\n",
       "San Francisco, CA                       207\n",
       "Canada, Moyen-Chari, Chad               205\n",
       "Toronto, Ontario, Canada                198\n",
       "Vancouver, British Columbia, Canada     190\n",
       "Brussels, Brussels Region, Belgium      188\n",
       "Berlin, Germany                         175\n",
       "Mississauga, Ontario, Canada            174\n",
       "Boston, MA                              170\n",
       "Bellevue, WA                            154\n",
       "dtype: int64"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.value_counts('Place').head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_pattern = r'(\\b\\w+[.]?(?=\\s*,|\\s*$))'\n",
    "def extract_country(location):\n",
    "    match = re.search(regex_pattern, location)\n",
    "    if match:\n",
    "        return match.group(1)\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "# apply the function to the 'Location' column of the DataFrame\n",
    "all_data['City'] = all_data['Place'].apply(extract_country)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords_lower = ['python','sql','r', 'tableau', 'power bi','azure','google cloud','databricks','git','aws','nosql','spark','docker','gcp','looker','etl','snowflake','redshift','excel','sas','javascript']\n",
    "#all_data['Description'].str.lower().str.contains(keywords_lower).astype(int)\n",
    "# iterate over the keywords and create a new column for each keyword\n",
    "for keyword in keywords_lower:\n",
    "    all_data[keyword] = all_data['Description'].str.lower().str.contains(keyword).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         True\n",
       "1         True\n",
       "2         True\n",
       "3        False\n",
       "4        False\n",
       "         ...  \n",
       "14631     True\n",
       "14632     True\n",
       "14633    False\n",
       "14634     True\n",
       "14635     True\n",
       "Name: Description, Length: 14636, dtype: bool"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data['Description'].str.lower().str.contains('python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job_ID</th>\n",
       "      <th>Link</th>\n",
       "      <th>Apply Link</th>\n",
       "      <th>Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Place</th>\n",
       "      <th>Description</th>\n",
       "      <th>HTML</th>\n",
       "      <th>Date</th>\n",
       "      <th>Salary</th>\n",
       "      <th>City</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3449267566</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/data-analys...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>Near</td>\n",
       "      <td>Pasadena, CA</td>\n",
       "      <td>Description\\n\\n\\n\\n\\nYou will be joining Near,...</td>\n",
       "      <td>&lt;div class=\"description__text description__tex...</td>\n",
       "      <td>2023-01-23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pasadena</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3424527214</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/data-analys...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>RennerBrown</td>\n",
       "      <td>New York, United States</td>\n",
       "      <td>*NO 3RD PARTIES*\\n\\nThe Analyst, Business Oper...</td>\n",
       "      <td>&lt;div class=\"description__text description__tex...</td>\n",
       "      <td>2023-01-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>York</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3422151601</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/data-analys...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>PointsBet</td>\n",
       "      <td>Denver, CO</td>\n",
       "      <td>Data Analyst\\n\\n\\n\\n\\nPointsBet (ASX: PBH) is ...</td>\n",
       "      <td>&lt;div class=\"description__text description__tex...</td>\n",
       "      <td>2023-01-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Denver</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3421351929</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/junior-data...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Junior Data Analyst - 100% Remote</td>\n",
       "      <td>Visionaire Partners</td>\n",
       "      <td>Norcross, GA</td>\n",
       "      <td>Data Analyst - Operations (Live Anywhere - mus...</td>\n",
       "      <td>&lt;div class=\"description__text description__tex...</td>\n",
       "      <td>2023-01-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Norcross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3451447956</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/data-analys...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>Spruce</td>\n",
       "      <td>Austin, TX</td>\n",
       "      <td>Who We Are\\n\\nAt Spruce, our mission is to cha...</td>\n",
       "      <td>&lt;div class=\"description__text description__tex...</td>\n",
       "      <td>2023-01-23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Austin</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Job_ID                                               Link  Apply Link  \\\n",
       "0  3449267566  https://www.linkedin.com/jobs/view/data-analys...         NaN   \n",
       "1  3424527214  https://www.linkedin.com/jobs/view/data-analys...         NaN   \n",
       "2  3422151601  https://www.linkedin.com/jobs/view/data-analys...         NaN   \n",
       "3  3421351929  https://www.linkedin.com/jobs/view/junior-data...         NaN   \n",
       "4  3451447956  https://www.linkedin.com/jobs/view/data-analys...         NaN   \n",
       "\n",
       "                               Title              Company  \\\n",
       "0                       Data Analyst                 Near   \n",
       "1                       Data Analyst          RennerBrown   \n",
       "2                       Data Analyst            PointsBet   \n",
       "3  Junior Data Analyst - 100% Remote  Visionaire Partners   \n",
       "4                       Data Analyst               Spruce   \n",
       "\n",
       "                      Place  \\\n",
       "0             Pasadena, CA    \n",
       "1  New York, United States    \n",
       "2               Denver, CO    \n",
       "3              Norcross, GA   \n",
       "4                Austin, TX   \n",
       "\n",
       "                                         Description  \\\n",
       "0  Description\\n\\n\\n\\n\\nYou will be joining Near,...   \n",
       "1  *NO 3RD PARTIES*\\n\\nThe Analyst, Business Oper...   \n",
       "2  Data Analyst\\n\\n\\n\\n\\nPointsBet (ASX: PBH) is ...   \n",
       "3  Data Analyst - Operations (Live Anywhere - mus...   \n",
       "4  Who We Are\\n\\nAt Spruce, our mission is to cha...   \n",
       "\n",
       "                                                HTML        Date  Salary  \\\n",
       "0  <div class=\"description__text description__tex...  2023-01-23     NaN   \n",
       "1  <div class=\"description__text description__tex...  2023-01-03     NaN   \n",
       "2  <div class=\"description__text description__tex...  2023-01-05     NaN   \n",
       "3  <div class=\"description__text description__tex...  2023-01-05     NaN   \n",
       "4  <div class=\"description__text description__tex...  2023-01-23     NaN   \n",
       "\n",
       "       City  \n",
       "0  Pasadena  \n",
       "1      York  \n",
       "2    Denver  \n",
       "3  Norcross  \n",
       "4    Austin  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_text = all_data['Description'].str.cat(sep=' ')\n",
    "# split string into list of words and count occurrences\n",
    "word_counts = pd.Series(all_text.split()).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\rafae\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "#stops = set(stopwords.words('english'))\n",
    "cachedStopWords = stopwords.words(\"english\")\n",
    "extra_stop_words = ['a', 'an', 'in', 'on']\n",
    "cachedStopWords += extra_stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "for stop_word in cachedStopWords:\n",
    "    word_counts = word_counts.drop(stop_word, errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_counts.to_excel('word_counts.xlsx', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import spacy\n",
    "\n",
    "languages = pd.read_csv('languages.csv')\n",
    "platforms = pd.read_csv('platforms.csv')\n",
    "databases = pd.read_csv('databases.csv')\n",
    "frameworks_tools = pd.read_csv('frameworks_tools_etc.csv')\n",
    "\n",
    "patterns = []\n",
    "for x in languages.name.tolist():\n",
    "    patterns.append({\"label\": \"PROG_LANG\", \"pattern\": [{\"lower\": w.lower()} for w in str(x).split()],\"id\": \"SKILLS\"})\n",
    "\n",
    "for x in databases.name.tolist():\n",
    "    patterns.append({\"label\": \"DB\",\"pattern\": [{\"lower\": w.lower()} for w in str(x).split()], \"id\": \"SKILLS\"})\n",
    "\n",
    "for x in platforms.name.tolist():\n",
    "    patterns.append({\"label\": \"PLATFORM\", \"pattern\": [{\"lower\": w.lower()} for w in str(x).split()], \"id\": \"SKILLS\"})\n",
    "\n",
    "for x in frameworks_tools.name.tolist():\n",
    "    patterns.append({\"label\": \"FRAMEWORKS\", \"pattern\": [{\"lower\": w.lower()} for w in str(x).split()], \"id\": \"SKILLS\"})\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "ruler = nlp.add_pipe(\"entity_ruler\", before=\"ner\")\n",
    "ruler.add_patterns(patterns)\n",
    "ruler.to_disk(\"patterns.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacy.pipeline.entityruler.EntityRuler at 0x28db4ccbf90>"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "ruler = nlp.add_pipe(\"entity_ruler\", before=\"ner\")\n",
    "patterns = \"patterns.jsonl\"\n",
    "ruler.from_disk(patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'PROG_LANG', 'text': 'Python'},\n",
       " {'label': 'PROG_LANG', 'text': 'R'},\n",
       " {'label': 'FRAMEWORKS', 'text': 'PyTorch'},\n",
       " {'label': 'FRAMEWORKS', 'text': 'TensorFlow'},\n",
       " {'label': 'FRAMEWORKS', 'text': 'NumPy'},\n",
       " {'label': 'FRAMEWORKS', 'text': 'Pandas'},\n",
       " {'label': 'FRAMEWORKS', 'text': 'Keras'},\n",
       " {'label': 'PROG_LANG', 'text': 'Julia'},\n",
       " {'label': 'DB', 'text': 'postgreSQL'},\n",
       " {'label': 'DB', 'text': 'MySQL'},\n",
       " {'label': 'DB', 'text': 'Snowflake'},\n",
       " {'label': 'PLATFORM', 'text': 'GCP'},\n",
       " {'label': 'PLATFORM', 'text': 'AWS'},\n",
       " {'label': 'FRAMEWORKS', 'text': 'PyTorch'},\n",
       " {'label': 'FRAMEWORKS', 'text': 'Tensorflow'}]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"\"\"\n",
    "Semrush\n",
    "is a leading digital marketing toolkit for SEO, PPC, and content marketing professionals worldwide, with 7 million marketers having already used Semrush to improve online marketing activities, monitor their website’s performance, as well as their competitors’.\n",
    "With over 45 tools within the software and search data for 140 countries, Semrush provides solutions for in-house marketing teams and agencies working with clients in any industry. Semrush is an absolute leader in helping good marketers become great!\n",
    "The Semrush\n",
    "Enterprise solutions unit\n",
    "is building new innovative solutions for large companies to unlock growth by answering recurring, complex, and unknown Digital Marketing-related questions (semi) automatically. This includes the collection, storage, and computation of huge amounts of data to create insights that are otherwise hard or impossible to get. Our solutions will connect data using machine learning to remove dependencies and friction for SEO, product, engineering, and marketing teams. New visually storytelling-like reports will help companies make better decisions about their Digital Marketing budgets.\n",
    "Joining our new unit means you will be working with us on the largest Digital Marketing data set available, and you will be part of shaping the Digital Marketing world with new solutions that are transforming how large enterprise companies do Digital Marketing. The teams will include product management, engineering, data science, product design, customer success, and marketing roles.\n",
    "For the new unit, the office will be in Berlin, but remote work (partially) is possible too.\n",
    "Key responsibilities\n",
    "Use analytical and statistical methods to analyze large amounts of digital marketing data, extracting actionable insights using advanced techniques such as data analysis, data mining, optimization tools, and machine learning.\n",
    "Develop predictive models based on historical data that provide future predictions of customer website performance.\n",
    "Validate and refine model assumptions with statistical analysis and testing.\n",
    "Build production-ready services that can be consumed by product teams.\n",
    "Technologies\n",
    "Python\n",
    "R\n",
    "Tydverse\n",
    "SQL\n",
    "PyTorch, TensorFlow, NumPy, Pandas, Keras, JAX, Julia\n",
    "postgreSQL, MySQL, Snowflake, GCP/AWS\n",
    "Java\n",
    "Job requirements\n",
    "3+ years experience with applied Machine/Deep Learning.\n",
    "Understanding of data structures, data modeling, and software architecture.\n",
    "Strong coding experience and proven ability to implement ML packages and neural network architectures using frameworks such as PyTorch or Tensorflow.\n",
    "Great communication skills and an ability to translate business context into data-oriented hypotheses to drive impact.\n",
    "BSc or BA degree in computer science, data science, applied math, or a related field.\n",
    "An appreciation for the connection between the product experience and models you build.\n",
    "Benefits\n",
    "This offer stands for the “FLEX” work format: some days a week you work from the office, and some #wfh. Trust us, it’s an efficient way of work. Proven!\n",
    "Flexible working day start that would suit a night owl and an early bird alike. You can start between 9:00 am and 12:00 pm.\n",
    "Private health insurance program. Life insurance.\n",
    "Agile approach to work (we’ll gladly teach you).\n",
    "Training/online courses and workshops/conferences/books to improve your hard and soft skills.\n",
    "Financial compensation for sports and hobbies (gym/dancing/languages/horseback riding/painting/wakeboard… and many more. It’s really up to you!)\n",
    "Corporate psychologist consultations. Mental health is as important as physical well-being.\n",
    "Awesome parties, team building, and corporate events in different formats – both online and offline, depending on the current global situation. We can have fun in any case!\n",
    "Semrush is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. We do not discriminate based on race, religion, creed, color, national origin, sex, pregnancy, sexual orientation, gender identity, gender expression, age, ancestry, physical or mental disability, or medical condition, including medical characteristics, genetic identity, marital status, military service, or any other classification protected by applicable local, state or federal laws. All employment decisions are based on business needs, job requirements, merit, and individual qualifications.\n",
    "\"\"\"\n",
    "doc = nlp(text)\n",
    "doc\n",
    "ents = [{'label': entity.label_, 'text': entity.text} for entity in doc.ents if entity.ent_id_ == 'SKILLS']\n",
    "ents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_tech(description):\n",
    "    doc = nlp(description)\n",
    "    frameworks, databases, platforms, prog_langs = [], [], [], []\n",
    "\n",
    "    for entity in doc.ents:\n",
    "        if entity.ent_id_ == 'SKILLS':\n",
    "            if entity.label_ == 'PROG_LANG' and entity.text not in prog_langs:\n",
    "                prog_langs.append(entity.text)\n",
    "            if entity.label_ == 'PLATFORM' and entity.text not in platforms:\n",
    "                platforms.append(entity.text)\n",
    "            if entity.label_ == 'DB' and entity.text not in databases:\n",
    "                databases.append(entity.text)\n",
    "            if entity.label_ == 'FRAMEWORKS' and entity.text not in frameworks:\n",
    "                frameworks.append(entity.text)\n",
    "\n",
    "    #prog_langs = \" \".join(prog_langs)\n",
    "    #platforms = \" \".join(platforms)\n",
    "    #databases = \" \".join(databases)\n",
    "    #frameworks = \" \".join(frameworks)\n",
    "    data = (prog_langs, platforms, databases, frameworks)\n",
    "    data = tuple([x for x in data if x])\n",
    "    if data:\n",
    "        return data\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data['keywords'] = all_data['Description'].apply(extract_tech)\n",
    "#all_data.value_counts('Place').head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    ([Python], [AWS], [Redshift], [Tableau, Pandas...\n",
       "1                      ([Python], [Tableau, Power BI])\n",
       "2    ([Python], [Azure Data Lake], [SQL Server], [D...\n",
       "3                              ([Access, SQL Server],)\n",
       "4                         ([Snowflake], [Looker, ETL])\n",
       "Name: keywords, dtype: object"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data['keywords'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job_ID</th>\n",
       "      <th>keywords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3449267566</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3449267566</td>\n",
       "      <td>AWS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3449267566</td>\n",
       "      <td>Redshift</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3449267566</td>\n",
       "      <td>Tableau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3449267566</td>\n",
       "      <td>Pandas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36225</th>\n",
       "      <td>3575199096</td>\n",
       "      <td>Power BI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36226</th>\n",
       "      <td>3575199096</td>\n",
       "      <td>Tableau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36227</th>\n",
       "      <td>3561496857</td>\n",
       "      <td>R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36228</th>\n",
       "      <td>3561496857</td>\n",
       "      <td>Python</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36229</th>\n",
       "      <td>3561496857</td>\n",
       "      <td>Power BI</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36230 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Job_ID  keywords\n",
       "0      3449267566    Python\n",
       "1      3449267566       AWS\n",
       "2      3449267566  Redshift\n",
       "3      3449267566   Tableau\n",
       "4      3449267566    Pandas\n",
       "...           ...       ...\n",
       "36225  3575199096  Power BI\n",
       "36226  3575199096   Tableau\n",
       "36227  3561496857         R\n",
       "36228  3561496857    Python\n",
       "36229  3561496857  Power BI\n",
       "\n",
       "[36230 rows x 2 columns]"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "teste = all_data[['Job_ID','keywords']].explode(\"keywords\").reset_index(drop=True)\n",
    "teste = teste[teste[\"keywords\"].notna()]\n",
    "teste[\"keywords\"] = teste[\"keywords\"].apply(lambda x: \",\".join(x))\n",
    "exploded_df = teste.assign(keywords=teste[\"keywords\"].str.split(\",\")).explode(\"keywords\").reset_index(drop=True)\n",
    "exploded_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.to_csv(\"jobs.csv\",index=False)\n",
    "exploded_df.to_csv(\"skills.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "alignmentgroup": "True",
         "hovertemplate": "%{x:,.2%}",
         "legendgroup": "",
         "marker": {
          "color": "#636efa",
          "pattern": {
           "shape": ""
          }
         },
         "name": "",
         "offsetgroup": "",
         "orientation": "h",
         "showlegend": false,
         "text": [
          "5.10%",
          "6.01%",
          "6.19%",
          "6.53%",
          "9.80%",
          "14.51%",
          "17.52%",
          "20.85%",
          "29.48%",
          "39.76%"
         ],
         "textposition": "auto",
         "type": "bar",
         "x": [
          0.05103853511888494,
          0.06012571740912818,
          0.06190215905985242,
          0.06531839300355288,
          0.09804591418420333,
          0.14512161792839573,
          0.17518447663295983,
          0.20852691992347636,
          0.29475266466247607,
          0.39758130636786004
         ],
         "xaxis": "x",
         "y": [
          "Oracle",
          "Looker",
          "Snowflake",
          "SAS",
          "AWS",
          "ETL",
          "R",
          "Power BI",
          "Tableau",
          "Python"
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "barmode": "relative",
        "legend": {
         "tracegroupgap": 0
        },
        "margin": {
         "b": 70,
         "l": 100,
         "r": 20,
         "t": 70
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Keyword Distribution in Job Ads"
        },
        "width": 700,
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "tickformat": ".2%",
         "title": {
          "text": "Percentage"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "Keywords"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "exploded_df = exploded_df[exploded_df[\"keywords\"] != \"Access\"]\n",
    "exploded_df[\"keywords\"].value_counts().head(10).values/len(all_data)\n",
    "\n",
    "counts = pd.DataFrame(exploded_df[\"keywords\"].value_counts())\n",
    "counts['pct'] = counts['keywords']/len(all_data)\n",
    "counts\n",
    "counts.head(10)\n",
    "counts = counts.head(10).iloc[::-1]\n",
    "#exploded_df[,]\n",
    "#px.bar(counts.head(10), x=counts.head(10).index, y=counts.head(10).values, labels={\"x\": \"Industry\", \"y\": \"Count\"})\n",
    "fig = px.bar(counts.head(10), x='pct', y=counts.head(10).index,text=counts.head(10)['pct'].apply(lambda x: f'{x:,.2%}'), orientation='h')\n",
    "\n",
    "# Customize the layout\n",
    "fig.update_layout(\n",
    "    title='Keyword Distribution in Job Ads',\n",
    "    xaxis_title='Percentage',\n",
    "    yaxis_title='Keywords',\n",
    "    margin=dict(l=100, r=20, t=70, b=70),\n",
    "    xaxis_tickformat=\".2%\",\n",
    "    width=700\n",
    ")\n",
    "fig.update_traces(hovertemplate='%{x:,.2%}')\n",
    "# Show the plot\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts.reset_index().to_csv(\"counts.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "alignmentgroup": "True",
         "hovertemplate": "%{x:,.2%}",
         "legendgroup": "",
         "marker": {
          "color": "#000001",
          "pattern": {
           "shape": ""
          }
         },
         "name": "",
         "offsetgroup": "",
         "orientation": "h",
         "showlegend": false,
         "text": [
          "5.10%",
          "6.01%",
          "6.19%",
          "6.53%",
          "9.80%",
          "14.51%",
          "17.52%",
          "20.85%",
          "29.48%",
          "39.76%"
         ],
         "textposition": "auto",
         "type": "bar",
         "x": [
          0.05103853511888494,
          0.06012571740912818,
          0.06190215905985242,
          0.06531839300355288,
          0.09804591418420333,
          0.14512161792839573,
          0.17518447663295983,
          0.20852691992347636,
          0.29475266466247607,
          0.39758130636786004
         ],
         "xaxis": "x",
         "y": [
          "Oracle",
          "Looker",
          "Snowflake",
          "SAS",
          "AWS",
          "ETL",
          "R",
          "Power BI",
          "Tableau",
          "Python"
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "barmode": "relative",
        "legend": {
         "tracegroupgap": 0
        },
        "margin": {
         "b": 70,
         "l": 100,
         "r": 20,
         "t": 70
        },
        "template": {
         "data": {
          "candlestick": [
           {
            "decreasing": {
             "line": {
              "color": "#000033"
             }
            },
            "increasing": {
             "line": {
              "color": "#000032"
             }
            },
            "type": "candlestick"
           }
          ],
          "contour": [
           {
            "colorscale": [
             [
              0,
              "#000011"
             ],
             [
              0.1111111111111111,
              "#000012"
             ],
             [
              0.2222222222222222,
              "#000013"
             ],
             [
              0.3333333333333333,
              "#000014"
             ],
             [
              0.4444444444444444,
              "#000015"
             ],
             [
              0.5555555555555556,
              "#000016"
             ],
             [
              0.6666666666666666,
              "#000017"
             ],
             [
              0.7777777777777778,
              "#000018"
             ],
             [
              0.8888888888888888,
              "#000019"
             ],
             [
              1,
              "#000020"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorscale": [
             [
              0,
              "#000011"
             ],
             [
              0.1111111111111111,
              "#000012"
             ],
             [
              0.2222222222222222,
              "#000013"
             ],
             [
              0.3333333333333333,
              "#000014"
             ],
             [
              0.4444444444444444,
              "#000015"
             ],
             [
              0.5555555555555556,
              "#000016"
             ],
             [
              0.6666666666666666,
              "#000017"
             ],
             [
              0.7777777777777778,
              "#000018"
             ],
             [
              0.8888888888888888,
              "#000019"
             ],
             [
              1,
              "#000020"
             ]
            ],
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorscale": [
             [
              0,
              "#000011"
             ],
             [
              0.1111111111111111,
              "#000012"
             ],
             [
              0.2222222222222222,
              "#000013"
             ],
             [
              0.3333333333333333,
              "#000014"
             ],
             [
              0.4444444444444444,
              "#000015"
             ],
             [
              0.5555555555555556,
              "#000016"
             ],
             [
              0.6666666666666666,
              "#000017"
             ],
             [
              0.7777777777777778,
              "#000018"
             ],
             [
              0.8888888888888888,
              "#000019"
             ],
             [
              1,
              "#000020"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "histogram2d": [
           {
            "colorscale": [
             [
              0,
              "#000011"
             ],
             [
              0.1111111111111111,
              "#000012"
             ],
             [
              0.2222222222222222,
              "#000013"
             ],
             [
              0.3333333333333333,
              "#000014"
             ],
             [
              0.4444444444444444,
              "#000015"
             ],
             [
              0.5555555555555556,
              "#000016"
             ],
             [
              0.6666666666666666,
              "#000017"
             ],
             [
              0.7777777777777778,
              "#000018"
             ],
             [
              0.8888888888888888,
              "#000019"
             ],
             [
              1,
              "#000020"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "icicle": [
           {
            "textfont": {
             "color": "white"
            },
            "type": "icicle"
           }
          ],
          "sankey": [
           {
            "textfont": {
             "color": "#000036"
            },
            "type": "sankey"
           }
          ],
          "scatter": [
           {
            "marker": {
             "line": {
              "width": 0
             }
            },
            "type": "scatter"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#000038"
             },
             "font": {
              "color": "#000037"
             },
             "line": {
              "color": "#000039"
             }
            },
            "header": {
             "fill": {
              "color": "#000040"
             },
             "font": {
              "color": "#000036"
             },
             "line": {
              "color": "#000039"
             }
            },
            "type": "table"
           }
          ],
          "waterfall": [
           {
            "connector": {
             "line": {
              "color": "#000036",
              "width": 2
             }
            },
            "decreasing": {
             "marker": {
              "color": "#000033"
             }
            },
            "increasing": {
             "marker": {
              "color": "#000032"
             }
            },
            "totals": {
             "marker": {
              "color": "#000034"
             }
            },
            "type": "waterfall"
           }
          ]
         },
         "layout": {
          "coloraxis": {
           "colorscale": [
            [
             0,
             "#000011"
            ],
            [
             0.1111111111111111,
             "#000012"
            ],
            [
             0.2222222222222222,
             "#000013"
            ],
            [
             0.3333333333333333,
             "#000014"
            ],
            [
             0.4444444444444444,
             "#000015"
            ],
            [
             0.5555555555555556,
             "#000016"
            ],
            [
             0.6666666666666666,
             "#000017"
            ],
            [
             0.7777777777777778,
             "#000018"
            ],
            [
             0.8888888888888888,
             "#000019"
            ],
            [
             1,
             "#000020"
            ]
           ]
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#000021"
            ],
            [
             0.1,
             "#000022"
            ],
            [
             0.2,
             "#000023"
            ],
            [
             0.3,
             "#000024"
            ],
            [
             0.4,
             "#000025"
            ],
            [
             0.5,
             "#000026"
            ],
            [
             0.6,
             "#000027"
            ],
            [
             0.7,
             "#000028"
            ],
            [
             0.8,
             "#000029"
            ],
            [
             0.9,
             "#000030"
            ],
            [
             1,
             "#000031"
            ]
           ],
           "sequential": [
            [
             0,
             "#000011"
            ],
            [
             0.1111111111111111,
             "#000012"
            ],
            [
             0.2222222222222222,
             "#000013"
            ],
            [
             0.3333333333333333,
             "#000014"
            ],
            [
             0.4444444444444444,
             "#000015"
            ],
            [
             0.5555555555555556,
             "#000016"
            ],
            [
             0.6666666666666666,
             "#000017"
            ],
            [
             0.7777777777777778,
             "#000018"
            ],
            [
             0.8888888888888888,
             "#000019"
            ],
            [
             1,
             "#000020"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#000011"
            ],
            [
             0.1111111111111111,
             "#000012"
            ],
            [
             0.2222222222222222,
             "#000013"
            ],
            [
             0.3333333333333333,
             "#000014"
            ],
            [
             0.4444444444444444,
             "#000015"
            ],
            [
             0.5555555555555556,
             "#000016"
            ],
            [
             0.6666666666666666,
             "#000017"
            ],
            [
             0.7777777777777778,
             "#000018"
            ],
            [
             0.8888888888888888,
             "#000019"
            ],
            [
             1,
             "#000020"
            ]
           ]
          },
          "colorway": [
           "#000001",
           "#000002",
           "#000003",
           "#000004",
           "#000005",
           "#000006",
           "#000007",
           "#000008",
           "#000009",
           "#000010"
          ]
         }
        },
        "title": {
         "text": "Keyword Distribution in Job Ads"
        },
        "width": 700,
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "tickformat": ".2%",
         "title": {
          "text": "Percstreamlit run appentage"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "Keywords"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "counts2 = pd.read_csv(\"counts.csv\")\n",
    "counts2.rename(columns={'Unnamed: 0':'skill'})\n",
    "ig = px.bar(counts2.head(10), x='pct', y=counts2.index,text=counts2['pct'].apply(lambda x: f'{x:,.2%}'), orientation='h')\n",
    "\n",
    "# Customize the layout\n",
    "fig.update_layout(\n",
    "    title='Keyword Distribution in Job Ads',\n",
    "    xaxis_title='Percstreamlit run appentage',\n",
    "    yaxis_title='Keywords',\n",
    "    margin=dict(l=100, r=20, t=70, b=70),\n",
    "    xaxis_tickformat=\".2%\",\n",
    "    width=700\n",
    ")\n",
    "fig.update_traces(hovertemplate='%{x:,.2%}')\n",
    "fig.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}