full_loop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from picamera.array import PiRGBArray\n",
    "from picamera import PiCamera"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "camera = PiCamera()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_fullscreen_window():\n",
    "    cv2.namedWindow(\"fw\", cv2.WND_PROP_FULLSCREEN)\n",
    "    cv2.setWindowProperty(\"fw\",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_image(im, timeout=1000):\n",
    "    im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)\n",
    "    cv2.imshow(\"fw\", im)\n",
    "    cv2.waitKey(timeout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanup_window():\n",
    "    cv2.destroyWindow(\"fw\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_image():\n",
    "    rawCapture = PiRGBArray(camera)\n",
    "    camera.capture(rawCapture, format=\"rgb\")\n",
    "    image = rawCapture.array\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scale_image(image):\n",
    "    small_image = cv2.resize(image[0:480, 160:640], dsize=(299,299))\n",
    "    small_image_f32 = np.float32(small_image)\n",
    "    small_image_f32_r3 = np.expand_dims(small_image_f32, 0)\n",
    "    scaled_image = small_image_f32_r3/255\n",
    "    return scaled_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "iv3 = keras.applications.InceptionV3(\n",
    "    include_top=True,\n",
    "    weights='imagenet',\n",
    "    input_tensor=None,\n",
    "    input_shape=None,\n",
    "    pooling=None,\n",
    "    classes=1000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_image(im):\n",
    "    prediction = iv3.predict(im)\n",
    "    return keras.applications.inception_v3.decode_predictions(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(im):\n",
    "    prediction = predict_image(im)\n",
    "    name = prediction[0][1][1]\n",
    "    return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "import shlex\n",
    "import tempfile\n",
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def say_something_better(text):\n",
    "    number, fname = tempfile.mkstemp(suffix='.wav')\n",
    "    subprocess.call([\"pico2wave\", \"--wave={}\".format(fname), \"-len-GB\", text])\n",
    "    subprocess.Popen([\"aplay\", \"{}\".format(fname)])\n",
    "    os.remove(fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('./joern_api_responses.json', 'r') as f:\n",
    "    thought_responses = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def category_to_thought(category):\n",
    "    retval = None\n",
    "    try:\n",
    "        retval = thought_responses[category]\n",
    "    except:\n",
    "        retval = [category, \"Beer\"]\n",
    "    return retval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def puttext(im, text, scale=1, thickness=1, top=True):\n",
    "    text = text.strip('. ')\n",
    "    h, w, _ = im.shape\n",
    "    x, y = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, scale, thickness)[0]\n",
    "    xpos = (w // 2) - (x // 2)\n",
    "    if top:\n",
    "        ypos = 30\n",
    "    else:\n",
    "        ypos = h - 10\n",
    "    im = cv2.putText(im, text, (xpos, ypos), cv2.FONT_HERSHEY_SIMPLEX, scale, (0, 0, 0), thickness+2)\n",
    "    im = cv2.putText(im, text, (xpos, ypos), cv2.FONT_HERSHEY_SIMPLEX, scale, (255, 255, 255), thickness)\n",
    "    return im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main_loop():\n",
    "    create_fullscreen_window()\n",
    "    for i in range(5):\n",
    "        im = get_image()\n",
    "        stimulus = predict(scale_image(im))\n",
    "        stimulus, response = category_to_thought(stimulus)\n",
    "        string_to_speak = \". When I see {}, I think of {}.\".format(stimulus, response)\n",
    "        string_to_speak = string_to_speak.replace(\"_\", \" \")\n",
    "        print(string_to_speak)\n",
    "        string_top, string_bottom = string_to_speak.split(', ')\n",
    "        im = puttext(im, string_top)\n",
    "        im = puttext(im, string_bottom, top=False)\n",
    "        show_image(im)\n",
    "        say_something_better(string_to_speak)\n",
    "        # time.sleep(10)\n",
    "    cleanup_window()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". When I see Spatula, I think of Film.\n",
      ". When I see Nipple, I think of Milk.\n",
      ". When I see Tub, I think of Technology.\n",
      ". When I see Band Aid, I think of 10.\n",
      ". When I see Bathtub, I think of House.\n"
     ]
    }
   ],
   "source": [
    "main_loop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}