notebooks/issue-30.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2021-11-15 01:40:00] Try to use the default NATS-Bench (topology) path from fast_mode=True and path=None.\n"
     ]
    }
   ],
   "source": [
    "from nats_bench import create\n",
    "import numpy as np\n",
    "\n",
    "# Create the API for the topology search space\n",
    "api = create(None, 'tss', fast_mode=True, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train-loss': 2.6261614637934385,\n",
       " 'train-accuracy': 33.934739619476986,\n",
       " 'train-per-time': 51.38146062405923,\n",
       " 'train-all-time': 10276.292124811845,\n",
       " 'valid-loss': 2.6819862944285076,\n",
       " 'valid-accuracy': 32.83333325195313,\n",
       " 'valid-per-time': 0.36819872515542285,\n",
       " 'valid-all-time': 73.63974503108457,\n",
       " 'test-loss': 2.7110290253957112,\n",
       " 'test-accuracy': 31.966666661580405,\n",
       " 'test-per-time': 0.36819872515542285,\n",
       " 'test-all-time': 73.63974503108457,\n",
       " 'valtest-loss': 2.6965076230367027,\n",
       " 'valtest-accuracy': 32.399999994913735,\n",
       " 'valtest-per-time': 0.7363974503108457,\n",
       " 'valtest-all-time': 147.27949006216915}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "name = 'ImageNet16-120'\n",
    "api.get_more_info(index=346, dataset=name, hp='200', iepoch=199, is_random=999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "can not find random seed (999) from [777, 888]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-9128042f185f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_more_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m347\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'200'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miepoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m199\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_random\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m999\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/nats_bench/api_topology.py\u001b[0m in \u001b[0;36mget_more_info\u001b[0;34m(self, index, dataset, iepoch, hp, is_random)\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mis_random\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseeds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;31m# collect the training information\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m         \u001b[0mtrain_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marchresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_metrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"train\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miepoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0miepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_random\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mis_random\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    221\u001b[0m         \u001b[0mtotal\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"iepoch\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         xinfo = {\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/nats_bench/api_utils.py\u001b[0m in \u001b[0;36mget_metrics\u001b[0;34m(self, dataset, setname, iepoch, is_random)\u001b[0m\n\u001b[1;32m    770\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_random\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# specify the seed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    771\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_random\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mx_seeds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 772\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"can not find random seed ({:}) from {:}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_random\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_seeds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    773\u001b[0m             \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx_seeds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_random\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    774\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minfos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: can not find random seed (999) from [777, 888]"
     ]
    }
   ],
   "source": [
    "api.get_more_info(index=347, dataset=name, hp='200', iepoch=199, is_random=999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}