loading_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How to use WoTe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting datasets\n",
      "  Downloading datasets-2.17.0-py3-none-any.whl.metadata (20 kB)\n",
      "Collecting filelock (from datasets)\n",
      "  Using cached filelock-3.13.1-py3-none-any.whl.metadata (2.8 kB)\n",
      "Collecting numpy>=1.17 (from datasets)\n",
      "  Downloading numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl.metadata (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.8/114.8 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pyarrow>=12.0.0 (from datasets)\n",
      "  Downloading pyarrow-15.0.0-cp311-cp311-macosx_11_0_arm64.whl.metadata (3.0 kB)\n",
      "Collecting pyarrow-hotfix (from datasets)\n",
      "  Using cached pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n",
      "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
      "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
      "Collecting pandas (from datasets)\n",
      "  Using cached pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl.metadata (19 kB)\n",
      "Collecting requests>=2.19.0 (from datasets)\n",
      "  Using cached requests-2.31.0-py3-none-any.whl.metadata (4.6 kB)\n",
      "Collecting tqdm>=4.62.1 (from datasets)\n",
      "  Downloading tqdm-4.66.2-py3-none-any.whl.metadata (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting xxhash (from datasets)\n",
      "  Using cached xxhash-3.4.1-cp311-cp311-macosx_11_0_arm64.whl.metadata (12 kB)\n",
      "Collecting multiprocess (from datasets)\n",
      "  Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n",
      "Collecting fsspec<=2023.10.0,>=2023.1.0 (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets)\n",
      "  Using cached fsspec-2023.10.0-py3-none-any.whl.metadata (6.8 kB)\n",
      "Collecting aiohttp (from datasets)\n",
      "  Downloading aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl.metadata (7.4 kB)\n",
      "Collecting huggingface-hub>=0.19.4 (from datasets)\n",
      "  Downloading huggingface_hub-0.20.3-py3-none-any.whl.metadata (12 kB)\n",
      "Requirement already satisfied: packaging in ./.venv/lib/python3.11/site-packages (from datasets) (23.2)\n",
      "Collecting pyyaml>=5.1 (from datasets)\n",
      "  Using cached PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl.metadata (2.1 kB)\n",
      "Collecting aiosignal>=1.1.2 (from aiohttp->datasets)\n",
      "  Using cached aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
      "Collecting attrs>=17.3.0 (from aiohttp->datasets)\n",
      "  Using cached attrs-23.2.0-py3-none-any.whl.metadata (9.5 kB)\n",
      "Collecting frozenlist>=1.1.1 (from aiohttp->datasets)\n",
      "  Using cached frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl.metadata (12 kB)\n",
      "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets)\n",
      "  Downloading multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl.metadata (4.2 kB)\n",
      "Collecting yarl<2.0,>=1.0 (from aiohttp->datasets)\n",
      "  Using cached yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl.metadata (31 kB)\n",
      "Collecting typing-extensions>=3.7.4.3 (from huggingface-hub>=0.19.4->datasets)\n",
      "  Using cached typing_extensions-4.9.0-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting charset-normalizer<4,>=2 (from requests>=2.19.0->datasets)\n",
      "  Using cached charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl.metadata (33 kB)\n",
      "Collecting idna<4,>=2.5 (from requests>=2.19.0->datasets)\n",
      "  Using cached idna-3.6-py3-none-any.whl.metadata (9.9 kB)\n",
      "Collecting urllib3<3,>=1.21.1 (from requests>=2.19.0->datasets)\n",
      "  Downloading urllib3-2.2.0-py3-none-any.whl.metadata (6.4 kB)\n",
      "Collecting certifi>=2017.4.17 (from requests>=2.19.0->datasets)\n",
      "  Downloading certifi-2024.2.2-py3-none-any.whl.metadata (2.2 kB)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n",
      "Collecting pytz>=2020.1 (from pandas->datasets)\n",
      "  Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n",
      "Collecting tzdata>=2022.7 (from pandas->datasets)\n",
      "  Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
      "Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "Downloading datasets-2.17.0-py3-none-any.whl (536 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.6/536.6 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hUsing cached fsspec-2023.10.0-py3-none-any.whl (166 kB)\n",
      "Downloading aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl (387 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m387.7/387.7 kB\u001b[0m \u001b[31m37.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading huggingface_hub-0.20.3-py3-none-any.whl (330 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.1/330.1 kB\u001b[0m \u001b[31m35.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl (14.0 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.0/14.0 MB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading pyarrow-15.0.0-cp311-cp311-macosx_11_0_arm64.whl (24.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.2/24.2 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0mm\n",
      "\u001b[?25hUsing cached PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl (167 kB)\n",
      "Using cached requests-2.31.0-py3-none-any.whl (62 kB)\n",
      "Downloading tqdm-4.66.2-py3-none-any.whl (78 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.3/78.3 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hUsing cached filelock-3.13.1-py3-none-any.whl (11 kB)\n",
      "Downloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl (11.8 MB)\n",
      "Using cached pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
      "Using cached xxhash-3.4.1-cp311-cp311-macosx_11_0_arm64.whl (30 kB)\n",
      "Using cached attrs-23.2.0-py3-none-any.whl (60 kB)\n",
      "Downloading certifi-2024.2.2-py3-none-any.whl (163 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m163.8/163.8 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl (118 kB)\n",
      "Using cached frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl (53 kB)\n",
      "Using cached idna-3.6-py3-none-any.whl (61 kB)\n",
      "Downloading multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl (30 kB)\n",
      "Downloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached typing_extensions-4.9.0-py3-none-any.whl (32 kB)\n",
      "Downloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading urllib3-2.2.0-py3-none-any.whl (120 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.9/120.9 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl (81 kB)\n",
      "Installing collected packages: pytz, xxhash, urllib3, tzdata, typing-extensions, tqdm, pyyaml, pyarrow-hotfix, numpy, multidict, idna, fsspec, frozenlist, filelock, dill, charset-normalizer, certifi, attrs, yarl, requests, pyarrow, pandas, multiprocess, aiosignal, huggingface-hub, aiohttp, datasets\n",
      "Successfully installed aiohttp-3.9.3 aiosignal-1.3.1 attrs-23.2.0 certifi-2024.2.2 charset-normalizer-3.3.2 datasets-2.17.0 dill-0.3.8 filelock-3.13.1 frozenlist-1.4.1 fsspec-2023.10.0 huggingface-hub-0.20.3 idna-3.6 multidict-6.0.5 multiprocess-0.70.16 numpy-1.26.4 pandas-2.2.0 pyarrow-15.0.0 pyarrow-hotfix-0.6 pytz-2024.1 pyyaml-6.0.1 requests-2.31.0 tqdm-4.66.2 typing-extensions-4.9.0 tzdata-2024.1 urllib3-2.2.0 xxhash-3.4.1 yarl-1.9.4\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating train split: 248 examples [00:00, 26561.82 examples/s]\n",
      "Generating test split: 412 examples [00:00, 33776.11 examples/s]\n",
      "Generating validation split: 169 examples [00:00, 28552.22 examples/s]\n"
     ]
    }
   ],
   "source": [
    "import datasets\n",
    "\n",
    "dataset = datasets.load_dataset('json', data_files={'train': 'data/train.json', 'test': 'data/test.json', \"validation\": 'data/validation.json'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['data_split', 'task_data', 'question_type', 'original_answer', 'question', 'domain', 'intent_question', 'conversation_id', 'extracted_answer', 'text_question', 'question_id', 'document_url', 'intent_answer', 'history'],\n",
       "        num_rows: 248\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['data_split', 'task_data', 'question_type', 'original_answer', 'question', 'domain', 'intent_question', 'conversation_id', 'extracted_answer', 'text_question', 'question_id', 'document_url', 'intent_answer', 'history'],\n",
       "        num_rows: 412\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['data_split', 'task_data', 'question_type', 'original_answer', 'question', 'domain', 'intent_question', 'conversation_id', 'extracted_answer', 'text_question', 'question_id', 'document_url', 'intent_answer', 'history'],\n",
       "        num_rows: 169\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data_split': 'train',\n",
       " 'task_data': \"{'title': 'Split Pea and Sweet Potato Soup', 'description': 'Cardamom and garam masala add subtle spice to split peas and sweet potatoes. This hearty soup can be enjoyed on its own or with warmed garlic naan for dipping.', 'ingredients': ['1 cup yellow split peas', '4 cups vegetable broth', '1 medium sweet potato, peeled and cut into 1/2-inch slices', '7 whole green cardamom pods', '1 tablespoon ghee (clarified butter) or canola oil', '1 medium yellow onion, chopped', '1 teaspoon garam masala', '1/2 teaspoon ground cumin', 'Lemon juice, to taste', 'Cayenne pepper, to taste', '1/8 teaspoon fine sea salt', '1/4 teaspoon ground black pepper'], 'steps': ['Stovetop method: Soak the peas in water to cover for at least 6 hours or overnight. Drain and rinse.', 'Heat ghee in a medium pot over medium heat.', 'Add onion, garam masala and cumin and cook until translucent and fragrant, about 10 minutes, stirring often.', 'Add the vegetable broth and bring to a boil.', 'Add soaked peas, sweet potato and cardamom pods.', 'Reduce heat to a simmer and cover.', 'Simmer until tender, about an hour.', 'Remove cardamom pods and season the soup with lemon, cayenne, salt, and pepper.', 'Slow cooker method: Place peas, broth, sweet potato and cardamom pods in a slow cooker.', 'Cook on high for 4 to 6 hours or on low for 6 to 8 hours, adding additional broth or water to thicken if needed.', 'When ready to serve, heat ghee in a skillet over medium heat.', 'Add onions, garam masala and cumin and cook 10 minutes, stirring often.', 'Add the spiced onions to the slow cooker.', 'Stir, cover and cook 15 minutes longer on high heat.', 'Remove cardamom pods and season the soup with lemon, cayenne and pepper.']}\",\n",
       " 'question_type': \"['factoid']\",\n",
       " 'original_answer': \"Now that it is covered and simmering, we are going to let go an hour. I've shared the step here.\",\n",
       " 'question': 'Alright, how long am I simmering this for?',\n",
       " 'domain': 'cooking',\n",
       " 'intent_question': 'ask_question_recipe_steps',\n",
       " 'conversation_id': 'Wizard-of-Task-food-15',\n",
       " 'extracted_answer': 'about an hour',\n",
       " 'text_question': 'Alright, how long am I simmering this for?',\n",
       " 'question_id': 'food-15-3',\n",
       " 'document_url': 'https://www.wholefoodsmarket.com/recipes/split-pea-and-sweet-potato-soup',\n",
       " 'intent_answer': 'request_next_step',\n",
       " 'history': \"student: This step is complete! Can you explain what's next? | teacher: Next, reduce the heat to a simmer and cover the pot. | student: Why do I need to cover the pot at this stage? | teacher: To prevent heat loss and minimize cooking time.\"}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train'][0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}