Skip to content

Commit

Permalink
Notebook format updates.
Browse files Browse the repository at this point in the history
  • Loading branch information
@joefernandez
joefernandez committed Dec 5, 2024
1 parent 0716c28 commit feda912
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 74 deletions.
43 changes: 10 additions & 33 deletions site/en/gemma/docs/paligemma/fine-tuning-paligemma.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,7 @@
"\n",
"To generate a Kaggle API key, open your [**Settings** page in Kaggle](https://www.kaggle.com/settings) and click **Create New Token**. This triggers the download of a `kaggle.json` file containing your API credentials.\n",
"\n",
"Then, in Colab, select **Secrets** (🔑) in the left pane and add your Kaggle username and Kaggle API key. Store your username under the name `KAGGLE_USERNAME` and your API key under the name `KAGGLE_KEY`.\n",
"\n"
"Then, in Colab, select **Secrets** (🔑) in the left pane and add your Kaggle username and Kaggle API key. Store your username under the name `KAGGLE_USERNAME` and your API key under the name `KAGGLE_KEY`.\n"
]
},
{
Expand Down Expand Up @@ -502,7 +501,7 @@
"\n",
" image = tf.constant(image)\n",
" image = tf.image.resize(image, (size, size), method='bilinear', antialias=True)\n",
" return image.numpy() / 127.5 - 1.0 # [0, 255]-\u003e[-1,1]\n",
" return image.numpy() / 127.5 - 1.0 # [0, 255]->[-1,1]\n",
"\n",
"def preprocess_tokens(prefix, suffix=None, seqlen=None):\n",
" # Model has been trained to handle tokenized text composed of a prefix with\n",
Expand Down Expand Up @@ -642,12 +641,12 @@
" return f\"data:image/jpeg;base64,{image_b64}\"\n",
"\n",
"def render_example(image, caption):\n",
" image = ((image + 1)/2 * 255).astype(np.uint8) # [-1,1] -\u003e [0, 255]\n",
" image = ((image + 1)/2 * 255).astype(np.uint8) # [-1,1] -> [0, 255]\n",
" return f\"\"\"\n",
" \u003cdiv style=\"display: inline-flex; align-items: center; justify-content: center;\"\u003e\n",
" \u003cimg style=\"width:128px; height:128px;\" src=\"{render_inline(image, resize=(64,64))}\" /\u003e\n",
" \u003cp style=\"width:256px; margin:10px; font-size:small;\"\u003e{html.escape(caption)}\u003c/p\u003e\n",
" \u003c/div\u003e\n",
" <div style=\"display: inline-flex; align-items: center; justify-content: center;\">\n",
" <img style=\"width:128px; height:128px;\" src=\"{render_inline(image, resize=(64,64))}\" />\n",
" <p style=\"width:256px; margin:10px; font-size:small;\">{html.escape(caption)}</p>\n",
" </div>\n",
" \"\"\"\n",
"\n",
"html_out = \"\"\n",
Expand Down Expand Up @@ -764,7 +763,7 @@
" # Append to html output.\n",
" for example, response in zip(examples, responses):\n",
" outputs.append((example[\"image\"], response))\n",
" if num_examples and len(outputs) \u003e= num_examples:\n",
" if num_examples and len(outputs) >= num_examples:\n",
" return outputs"
]
},
Expand Down Expand Up @@ -872,36 +871,14 @@
],
"metadata": {
"colab": {
"gpuType": "T4",
"last_runtime": {
"build_target": "//learning/grp/tools/ml_python:ml_notebook",
"kind": "private"
},
"private_outputs": true,
"provenance": [
{
"file_id": "17AiK8gRY7oiquQGkBH0d08PFQo3Kyx1I",
"timestamp": 1715287187925
},
{
"file_id": "1qZlJfPyfKRrNcz2shxQ93HnnE5Ge1LLn",
"timestamp": 1715019972450
},
{
"file_id": "1JFnlD2kSiTNexdPw_NYRtuW6uuSTI0kD",
"timestamp": 1714585741026
}
],
"name": "fine-tuning-paligemma.ipynb",
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
}
62 changes: 21 additions & 41 deletions site/en/gemma/docs/paligemma/inference-with-keras.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,17 @@
"source": [
"# Generate PaliGemma output with Keras\n",
"\n",
"\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
"\u003ctd\u003e\n",
"\u003ca target=\"_blank\" href=\"https://ai.google.dev/gemma/docs/paligemma/fine-tuning-paligemma\"\u003e\u003cimg src=\"https://ai.google.dev/static/site-assets/images/docs/notebook-site-button.png\" height=\"32\" width=\"32\" /\u003eView on ai.google.dev\u003c/a\u003e\n",
"\u003c/td\u003e\n",
"\u003ctd\u003e\n",
"\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/google/generative-ai-docs/blob/main/site/en/gemma/docs/paligemma/inference-with-keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
"\u003c/td\u003e\n",
"\u003ctd\u003e\n",
"\u003ca target=\"_blank\" href=\"https://github.com/google/generative-ai-docs/blob/main/site/en/gemma/docs/paligemma/inference-with-keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
"\u003c/td\u003e\n",
"\u003c/table\u003e"
"<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
"<td>\n",
"<a target=\"_blank\" href=\"https://ai.google.dev/gemma/docs/paligemma/fine-tuning-paligemma\"><img src=\"https://ai.google.dev/static/site-assets/images/docs/notebook-site-button.png\" height=\"32\" width=\"32\" />View on ai.google.dev</a>\n",
"</td>\n",
"<td>\n",
"<a target=\"_blank\" href=\"https://colab.research.google.com/github/google/generative-ai-docs/blob/main/site/en/gemma/docs/paligemma/inference-with-keras.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
"</td>\n",
"<td>\n",
"<a target=\"_blank\" href=\"https://github.com/google/generative-ai-docs/blob/main/site/en/gemma/docs/paligemma/inference-with-keras.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
"</td>\n",
"</table>"
]
},
{
Expand Down Expand Up @@ -278,8 +278,7 @@
"* **`read_img`:** Helper method for `read_img_from_url`. This method is what actually opens the image, resizes it so that it fits in the model's constraints, and puts it into an array that can be interpreted by the model.\n",
"* **`read_img_from_url`:** Takes in an image via a valid URL. You need this method to pass the image to the model.\n",
"\n",
"You'll use `read_img_from_url` in the next step of this notebook.\n",
"\n"
"You'll use `read_img_from_url` in the next step of this notebook.\n"
]
},
{
Expand Down Expand Up @@ -310,8 +309,8 @@
"\n",
"def parse_bbox_and_labels(detokenized_output: str):\n",
" matches = re.finditer(\n",
" '\u003cloc(?P\u003cy0\u003e\\d\\d\\d\\d)\u003e\u003cloc(?P\u003cx0\u003e\\d\\d\\d\\d)\u003e\u003cloc(?P\u003cy1\u003e\\d\\d\\d\\d)\u003e\u003cloc(?P\u003cx1\u003e\\d\\d\\d\\d)\u003e'\n",
" ' (?P\u003clabel\u003e.+?)( ;|$)',\n",
" '<loc(?P<y0>\\d\\d\\d\\d)><loc(?P<x0>\\d\\d\\d\\d)><loc(?P<y1>\\d\\d\\d\\d)><loc(?P<x1>\\d\\d\\d\\d)>'\n",
" ' (?P<label>.+?)( ;|$)',\n",
" detokenized_output,\n",
" )\n",
" labels, boxes = [], []\n",
Expand Down Expand Up @@ -362,7 +361,7 @@
" mask = Image.fromarray(mask)\n",
" mask = mask.resize((x2 - x1, y2 - y1), resample=Image.NEAREST)\n",
" mask = np.array(mask)\n",
" binary_mask = (mask \u003e 0.5).astype(np.uint8)\n",
" binary_mask = (mask > 0.5).astype(np.uint8)\n",
"\n",
"\n",
" # Place the binary mask onto the full mask\n",
Expand All @@ -373,7 +372,7 @@
" if isinstance(image, Image.Image):\n",
" image = np.array(image)\n",
" blended_image = image.copy()\n",
" mask_indices = full_mask \u003e 0\n",
" mask_indices = full_mask > 0\n",
" alpha = 0.5\n",
"\n",
" for c in range(3):\n",
Expand Down Expand Up @@ -568,10 +567,10 @@
"source": [
"import big_vision.evaluators.proj.paligemma.transfers.segmentation as segeval\n",
"reconstruct_masks = segeval.get_reconstruct_masks('oi')\n",
"def parse_segments(detokenized_output: str) -\u003e tuple[np.ndarray, np.ndarray]:\n",
"def parse_segments(detokenized_output: str) -> tuple[np.ndarray, np.ndarray]:\n",
" matches = re.finditer(\n",
" '\u003cloc(?P\u003cy0\u003e\\d\\d\\d\\d)\u003e\u003cloc(?P\u003cx0\u003e\\d\\d\\d\\d)\u003e\u003cloc(?P\u003cy1\u003e\\d\\d\\d\\d)\u003e\u003cloc(?P\u003cx1\u003e\\d\\d\\d\\d)\u003e'\n",
" + ''.join(f'\u003cseg(?P\u003cs{i}\u003e\\d\\d\\d)\u003e' for i in range(16)),\n",
" '<loc(?P<y0>\\d\\d\\d\\d)><loc(?P<x0>\\d\\d\\d\\d)><loc(?P<y1>\\d\\d\\d\\d)><loc(?P<x1>\\d\\d\\d\\d)>'\n",
" + ''.join(f'<seg(?P<s{i}>\\d\\d\\d)>' for i in range(16)),\n",
" detokenized_output,\n",
" )\n",
" boxes, segs = [], []\n",
Expand Down Expand Up @@ -673,33 +672,14 @@
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "T4",
"machine_shape": "hm",
"private_outputs": true,
"provenance": [
{
"file_id": "1Mh7SHaP1cd5XuuHLmprKoBfr9ddSVu3a",
"timestamp": 1715369116355
},
{
"file_id": "1Xo8MY-GJsjjBKS1l_hQ8AImMNw9yxqmH",
"timestamp": 1715208696058
},
{
"file_id": "1bLQYoesGy8awiA0UT0askWIjIqcivfdY",
"timestamp": 1715147134742
}
],
"name": "inference-with-keras.ipynb",
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
}

0 comments on commit feda912

Please sign in to comment.