diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..fc3d6f1 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,29 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux/.devcontainer/base.Dockerfile +FROM mcr.microsoft.com/vscode/devcontainers/universal:2-focal + +RUN apt-get update && apt-get -y install --no-install-recommends \ + ffmpeg \ + python3.8-venv \ + gcc \ + pciutils + +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#install-guide +RUN distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ + && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + +RUN apt-get update \ +&& apt-get install -y nvidia-docker2 \ +&& systemctl restart docker + +#create global virtual environment using python standard library tools of virtualenv +ARG USER="codespace" +ARG VENV_PATH="/home/${USER}/venv" +COPY requirements.txt /tmp/ +COPY Makefile /tmp/ +RUN su $USER -c "/usr/bin/python3 -m venv /home/${USER}/venv" \ + && su $USER -c "${VENV_PATH}/bin/pip --disable-pip-version-check --no-cache-dir install -r /tmp/requirements.txt" \ + && rm -rf /tmp/requirements.txt + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..eeffdd5 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,81 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux +{ + "name": "GitHub Codespaces (Default)", + + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "features": { + "ghcr.io/devcontainers/features/nvidia-cuda:1": { + "installCudnn": true + }, + "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {} + }, + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "go.toolsManagement.checkForUpdates": "local", + "go.useLanguageServer": true, + "go.gopath": "/go", + "python.defaultInterpreterPath": "/home/codespace/venv/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/home/codespace/venv/bin/autopep8", + "python.formatting.blackPath": "/home/codespace/venv/bin/black", + "python.formatting.yapfPath": "/home/codespace/venv/bin/yapf", + "python.linting.banditPath": "/home/codespace/venv/bin/bandit", + "python.linting.flake8Path": "/home/codespace/venv/bin/flake8", + "python.linting.mypyPath": "/home/codespace/venv/bin/mypy", + "python.linting.pycodestylePath": "/home/codespace/venv/bin/pycodestyle", + "python.linting.pydocstylePath": "/home/codespace/venv/bin/pydocstyle", + "python.linting.pylintPath": "/home/codespace/venv/bin/pylint", + "lldb.executable": "/usr/bin/lldb", + "files.watcherExclude": { + "**/target/**": true + } + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "GitHub.vscode-pull-request-github", + "GitHub.copilot-nightly", + "GitHub.copilot-labs", + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-python.vscode-pylance", + "ms-python.python", + "ms-vscode.makefile-tools" + ] + } + }, + + "remoteUser": "codespace", + + "overrideCommand": false, + + "mounts": ["source=codespaces-linux-var-lib-docker,target=/var/lib/docker,type=volume"], + + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + "--privileged", + "--init", + //"--gpus", "all" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // "oryx build" will automatically install your dependencies and attempt to build your project + //"postCreateCommand": "oryx build -p virtualenv_name=.venv --log-file /tmp/oryx-build.log --manifest-dir /tmp || echo 'Could not auto-build. Skipping.'" + "postCreateCommand": "bash setup.sh" +} diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml new file mode 100644 index 0000000..0d706ac --- /dev/null +++ b/.github/workflows/cicd.yml @@ -0,0 +1,23 @@ +name: CI +on: + push: + branches: [ "GPU" ] + pull_request: + branches: [ "GPU" ] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: install packages + run: make install + - name: lint + run: make lint + - name: test + run: make test + - name: format + run: make format + - name: deploy + run: make deploy diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..763a95f --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,18 @@ +name: Docker Image CI + +on: + pull_request: + branches: [ "main" ] + # Allow mannually trigger + workflow_dispatch: + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Build the Codespaces container image + run: docker build . --file .devcontainer/Dockerfile diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml new file mode 100644 index 0000000..8fccd65 --- /dev/null +++ b/.github/workflows/static.yml @@ -0,0 +1,42 @@ +# Simple workflow for deploying static content to GitHub Pages +name: Deploy static content to Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["GPU"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + # Single deploy job since we're just deploying + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Pages + uses: actions/configure-pages@v2 + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + with: + # Upload entire repository + path: '.' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..00525fa --- /dev/null +++ b/.gitignore @@ -0,0 +1,143 @@ +#ignore big media files +media +#ignore transcript changes from whisper tests:wq +four-score.m4a.* + +#ignore huggingface +summarizeApp +#ignore fine-tuning +test_trainer/ + +#ignore pytorch artifacts +data +model.pth + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0be43ed --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:latest +RUN apk update && apk add bash + +WORKDIR /app +COPY repeat.sh /app \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9e841e7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..db7c921 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +install: + pip install --upgrade pip &&\ + pip install -r requirements.txt + #force install latest whisper + pip install --upgrade --no-deps --force-reinstall git+https://github.com/openai/whisper.git +test: + python -m pytest -vv --cov=main --cov=mylib test_*.py + +format: + black *.py hugging-face/zero_shot_classification.py hugging-face/hf_whisper.py + +lint: + pylint --disable=R,C --ignore-patterns=test_.*?py *.py mylib/*.py\ + hugging-face/zero_shot_classification.py hugging-face/hf_whisper.py + +container-lint: + docker run --rm -i hadolint/hadolint < Dockerfile + +checkgpu: + echo "Checking GPU for PyTorch" + python utils/verify_pytorch.py + echo "Checking GPU for Tensorflow" + python utils/verify_tf.py + +refactor: format lint + +deploy: + #deploy goes here + +all: install lint test format deploy diff --git a/README.md b/README.md new file mode 100644 index 0000000..683c2ea --- /dev/null +++ b/README.md @@ -0,0 +1,110 @@ +[![CI](https://github.com/nogibjj/mlops-template/actions/workflows/cicd.yml/badge.svg?branch=GPU)](https://github.com/nogibjj/mlops-template/actions/workflows/cicd.yml) +[![Codespaces Prebuilds](https://github.com/nogibjj/mlops-template/actions/workflows/codespaces/create_codespaces_prebuilds/badge.svg?branch=GPU)](https://github.com/nogibjj/mlops-template/actions/workflows/codespaces/create_codespaces_prebuilds) + +## Template for MLOPs projects with GPU + +**CONDA IS NOT NEEDED AS A PACKAGE MANAGER. All setup is done using the Python Software Foundation recommended tools: virtualenv and pip and mainstream production tools Docker. Please see [PEP 453](https://peps.python.org/pep-0453/) "officially recommend the use of pip as the default installer for Python packages"** + +*GitHub Codespaces are FREE for education and as are GPU Codespaces as of this writing in December 2022* + +1. First thing to do on launch is to open a new shell and verify virtualenv is sourced. + +Things included are: + +* `Makefile` + +* `Pytest` + +* `pandas` + +* `Pylint` or `ruff` + +* `Dockerfile` + +* `GitHub copilot` + +* `jupyter` and `ipython` + +* Most common Python libraries for ML/DL and Hugging Face + +* `githubactions` + +## Two fun tools to explore: + +* Zero-shot classification: ./hugging-face/zero_shot_classification.py classify +* Yake for candidate label creation: ./utils/kw_extract.py + +## Try out Bento + +* [tutorial bento](https://docs.bentoml.org/en/latest/tutorial.html) + +`docker run -it --rm -p 8888:8888 -p 3000:3000 -p 3001:3001 bentoml/quickstart:latest` + +### Verify GPU works + +The following examples test out the GPU (including Docker GPU) + +* run pytorch training test: `python utils/quickstart_pytorch.py` +* run pytorch CUDA test: `python utils/verify_cuda_pytorch.py` +* run tensorflow training test: `python utils/quickstart_tf2.py` +* run nvidia monitoring test: `nvidia-smi -l 1` it should show a GPU +* run whisper transcribe test `./utils/transcribe-whisper.sh` and verify GPU is working with `nvidia-smi -l 1` +* run `lspci | grep -i nvidia` you should see something like: `0001:00:00.0 3D controller: NVIDIA Corporation GV100GL [Tesla V100 PCIe 16GB] (rev a1)` + + +Additionally, this workspace is setup to fine-tune Hugging Face + +![fine-tune](https://user-images.githubusercontent.com/58792/195709866-121f994e-3531-493b-99af-c3266c4e28ea.jpg) + + +`python hugging-face/hf_fine_tune_hello_world.py` + +#### Verify containerized GPU works for Tensorflow + +*Because of potential versioning conflicts between PyTorch and Tensorflow it is recommended to run Tensorflow via GPU Container and PyTorch via default environment.* + +See [TensorFlow GPU documentation](https://www.tensorflow.org/install/docker) +* Run `docker run --gpus all -it --rm tensorflow/tensorflow:latest-gpu \ + python -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"` + +* Also interactively explore: `docker run --gpus all -it --rm tensorflow/tensorflow:latest-gpu`, then when inside run: +`apt-get update && apt-get install pciutils` then `lspci | grep -i nvidia` + +* To mount the code into your container: `docker run --gpus all -it --rm -v $(pwd):/tmp tensorflow/tensorflow:latest-gpu /bin/bash`. Then do `apt-get install -y git && cd /tmp`. Then all you need to do is run `make install`. Now you can verify you can train deep learning models by doing `python utils/quickstart_tf2.py` + +##### More Tensorflow GPU Ideas + +https://www.tensorflow.org/resources/recommendation-systems + +```bash +# Deploy the retrieval model with TensorFlow Serving +docker run -t --rm -p 8501:8501 \ + -v "RETRIEVAL/MODEL/PATH:/models/retrieval" \ + -e MODEL_NAME=retrieval tensorflow/serving & +``` + +### Setup Docker Toolkit NVidia + +* [reference docs](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#install-guide) + +![mlops-tensorflow-gpu](https://user-images.githubusercontent.com/58792/206875904-114b4cf0-879d-497b-8690-267dac8b222d.jpg) + + +### Used in Following Projects + +Used as the base and customized in the following Duke MLOps and Applied Data Engineering Coursera Labs: + +* [MLOPs-C2-Lab1-CICD](https://github.com/nogibjj/Coursera-MLOPs-Foundations-Lab-1-CICD) +* [MLOps-C2-Lab2-PokerSimulator](https://github.com/nogibjj/Coursera-MLOPs-Foundations-Lab-2-poker-simulator) +* [MLOps-C2-Final-HuggingFace](https://github.com/nogibjj/Coursera-MLOps-C2-Final-HuggingFace) +* [Coursera-MLOps-C2-lab3-probability-simulations](Coursera-MLOps-C2-lab3-probability-simulations) +* [Coursera-MLOps-C2-lab4-greedy-optimization](https://github.com/nogibjj/Coursera-MLOps-C2-lab4-greedy-optimization) +### References + +* [Docker "one-liners" for Tensorflow recommenders](https://www.tensorflow.org/resources/recommendation-systems) +* [Watch GitHub Universe Talk: Teaching MLOps at scale with Github](https://watch.githubuniverse.com/on-demand/ec17cbb3-0a89-4764-90a5-9debb58515f8) +* [Building Cloud Computing Solutions at Scale Specialization](https://www.coursera.org/specializations/building-cloud-computing-solutions-at-scale) +* [Python, Bash and SQL Essentials for Data Engineering Specialization](https://www.coursera.org/learn/web-app-command-line-tools-for-data-engineering-duke) +* [Implementing MLOps in the Enterprise](https://learning.oreilly.com/library/view/implementing-mlops-in/9781098136574/) +* [Practical MLOps: Operationalizing Machine Learning Models](https://www.amazon.com/Practical-MLOps-Operationalizing-Machine-Learning/dp/1098103017) +* [Coursera-Dockerfile](https://gist.github.com/noahgift/82a34d56f0a8f347865baaa685d5e98d) diff --git a/bentoml/README.md b/bentoml/README.md new file mode 100644 index 0000000..e8da322 --- /dev/null +++ b/bentoml/README.md @@ -0,0 +1,36 @@ +## Bento Exploration + +`bentoml models list` + +### Very clean format for dealing with models + +`ls /home/codespace/bentoml/models/iris_clf/ecde3adktowieaan/` +`model.yaml` and `saved_model.pkl` + +### Export the model + +`bentoml models export iris_clf:latest .` + +### Try out Yatai + +* Must install minikube + +https://github.com/bentoml/Yatai + +`minikube start --cpus 4 --memory 4096` + +Check if it is running: + +`minikube status` + +Double check context: + +`kubectl config current-context` + +Enable ingress + +`minikube addons enable ingress` + +Then install: + +`bash <(curl -s "https://raw.githubusercontent.com/bentoml/yatai/main/scripts/quick-install-yatai.sh")` \ No newline at end of file diff --git a/bentoml/hello.py b/bentoml/hello.py new file mode 100644 index 0000000..19652cf --- /dev/null +++ b/bentoml/hello.py @@ -0,0 +1,16 @@ +import bentoml + +from sklearn import svm +from sklearn import datasets + +# Load training data set +iris = datasets.load_iris() +X, y = iris.data, iris.target + +# Train the model +clf = svm.SVC(gamma='scale') +clf.fit(X, y) + +# Save model to the BentoML local model store +saved_model = bentoml.sklearn.save_model("iris_clf", clf) +print(f"Model saved: {saved_model}") diff --git a/examples/quickstart/.bentoignore b/examples/quickstart/.bentoignore new file mode 100644 index 0000000..f4e455d --- /dev/null +++ b/examples/quickstart/.bentoignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +*$py.class +.ipynb_checkpoints diff --git a/examples/quickstart/BENCHMARK.md b/examples/quickstart/BENCHMARK.md new file mode 100644 index 0000000..d822c15 --- /dev/null +++ b/examples/quickstart/BENCHMARK.md @@ -0,0 +1,12 @@ +Run the iris_classifier service in production mode: + +| Protocol | Command | +| -------- | -------------------------------------------------------- | +| HTTP | `bentoml serve-http iris_classifier:latest --production` | +| gRPC | `bentoml serve-grpc iris_classifier:latest --production` | + +Start locust testing client: + +```bash +locust --class-picker -H http://localhost:3000 +``` diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md new file mode 100644 index 0000000..74edb97 --- /dev/null +++ b/examples/quickstart/README.md @@ -0,0 +1,191 @@ +# BentoML Scikit-Learn Tutorial + +This is a sample project demonstrating basic usage of [BentoML](https://github.com/bentoml) with +Scikit-learn. + +In this project, we will train a classifier model using Scikit-learn and the Iris dataset, build +an prediction service for serving the trained model via an HTTP server, and containerize the +model server as a docker image for production deployment. + +This project is also available to run from a notebook: https://github.com/bentoml/BentoML/blob/main/examples/quickstart/iris_classifier.ipynb + +### Install Dependencies + +Install python packages required for running this project: +```bash +pip install -r ./requirements.txt +``` + +### Model Training + +First step, train a classification model with sklearn's built-in iris dataset and save the model +with BentoML: + +```bash +import bentoml +from sklearn import svm, datasets + +# Load training data +iris = datasets.load_iris() +X, y = iris.data, iris.target + +# Model Training +clf = svm.SVC() +clf.fit(X, y) + +# Save model to BentoML local model store +bentoml.sklearn.save_model("iris_clf", clf) +``` + +This will save a new model in the BentoML local model store, a new version tag is automatically +generated when the model is saved. You can see all model revisions from CLI via `bentoml models` +commands: + +```bash +bentoml models get iris_clf:latest + +bentoml models list + +bentoml models --help +``` + +To verify that the saved model can be loaded correctly, run the following: + +```python +import bentoml + +loaded_model = bentoml.sklearn.load_model("iris_clf:latest") + +loaded_model.predict([[5.9, 3. , 5.1, 1.8]]) # => array(2) +``` + +In BentoML, the recommended way of running ML model inference in serving is via Runner, which +gives BentoML more flexibility in terms of how to schedule the inference computation, how to +batch inference requests and take advantage of hardware resoureces available. Saved models can +be loaded as Runner instance as shown below: + +```python +import bentoml + +# Create a Runner instance: +iris_clf_runner = bentoml.sklearn.get("iris_clf:latest").to_runner() + +# Runner#init_local initializes the model in current process, this is meant for development and testing only: +iris_clf_runner.init_local() + +# This should yield the same result as the loaded model: +iris_clf_runner.predict.run([[5.9, 3., 5.1, 1.8]]) +``` + + +### Serving the model + +A simple BentoML Service that serves the model saved above look like this: + +```python +import numpy as np +import bentoml +from bentoml.io import NumpyNdarray + +iris_clf_runner = bentoml.sklearn.get("iris_clf:latest").to_runner() + +svc = bentoml.Service("iris_classifier", runners=[iris_clf_runner]) + +@svc.api(input=NumpyNdarray(), output=NumpyNdarray()) +async def classify(input_series: np.ndarray) -> np.ndarray: + return await iris_clf_runner.predict.async_run(input_series) +``` + +Copy this to a `service.py` file, and run your service with Bento Server locally: + +```bash +bentoml serve service.py:svc --reload +``` + +Open your web browser at http://127.0.0.1:3000 to view the Bento UI for sending test requests. + +You may also send request with `curl` command or any HTTP client, e.g.: + +```bash +curl -X POST -H "content-type: application/json" --data "[[5.9, 3, 5.1, 1.8]]" http://127.0.0.1:3000/classify +``` + + +### Build Bento for deployment + +Bento is the distribution format in BentoML which captures all the source code, model files, config +files and dependency specifications required for running the service for production deployment. Think +of it as Docker/Container designed for machine learning models. + +To begin with building Bento, create a `bentofile.yaml` under your project directory: + +```yaml +service: "service.py:svc" +labels: + owner: bentoml-team + project: gallery +include: +- "*.py" +python: + packages: + - scikit-learn + - pandas +``` + +Next, run `bentoml build` from current directory to start the Bento build: + +``` +> bentoml build + +05/05/2022 19:19:16 INFO [cli] Building BentoML service "iris_classifier:5wtigdwm4kwzduqj" from build context "/Users/bentoml/workspace/gallery/quickstart" +05/05/2022 19:19:16 INFO [cli] Packing model "iris_clf:4i7wbngm4crhpuqj" from "/Users/bentoml/bentoml/models/iris_clf/4i7wbngm4crhpuqj" +05/05/2022 19:19:16 INFO [cli] Successfully saved Model(tag="iris_clf:4i7wbngm4crhpuqj", + path="/var/folders/bq/gdsf0kmn2k1bf880r_l238600000gn/T/tmp26dx354ubentoml_bento_iris_classifier/models/iris_clf/4i7wbngm4crhpuqj/") +05/05/2022 19:19:16 INFO [cli] Locking PyPI package versions.. +05/05/2022 19:19:17 INFO [cli] + ██████╗░███████╗███╗░░██╗████████╗░█████╗░███╗░░░███╗██╗░░░░░ + ██╔══██╗██╔════╝████╗░██║╚══██╔══╝██╔══██╗████╗░████║██║░░░░░ + ██████╦╝█████╗░░██╔██╗██║░░░██║░░░██║░░██║██╔████╔██║██║░░░░░ + ██╔══██╗██╔══╝░░██║╚████║░░░██║░░░██║░░██║██║╚██╔╝██║██║░░░░░ + ██████╦╝███████╗██║░╚███║░░░██║░░░╚█████╔╝██║░╚═╝░██║███████╗ + ╚═════╝░╚══════╝╚═╝░░╚══╝░░░╚═╝░░░░╚════╝░╚═╝░░░░░╚═╝╚══════╝ + +05/05/2022 19:19:17 INFO [cli] Successfully built Bento(tag="iris_classifier:5wtigdwm4kwzduqj") at "/Users/bentoml/bentoml/bentos/iris_classifier/5wtigdwm4kwzduqj/" +``` + +A new Bento is now built and saved to local Bento store. You can view and manage it via +`bentoml list`,`bentoml get` and `bentoml delete` CLI command. + + +### Containerize and Deployment + +Bento is designed to be deployed to run efficiently in a variety of different environments. +And there are lots of deployment options and tools as part of the BentoML eco-system, such as +[Yatai](https://github.com/bentoml/Yatai) and [bentoctl](https://github.com/bentoml/bentoctl) for +direct deployment to cloud platforms. + +In this guide, we will show you the most basic way of deploying a Bento, which is converting a Bento +into a Docker image containing the HTTP model server. + +Make sure you have docker installed and docker deamon running, and run the following commnand: + +```bash +bentoml containerize iris_classifier:latest +``` + +This will build a new docker image with all source code, model files and dependencies in place, +and ready for production deployment. To start a container with this docker image locally, run: + +```bash +docker run -p 3000:3000 iris_classifier:invwzzsw7li6zckb2ie5eubhd +``` + +## What's Next? + +- 👉 [Pop into our Slack community!](https://l.linklyhq.com/l/ktO8) We're happy to help with any issue you face or even just to meet you and hear what you're working on. +- Dive deeper into the [Core Concepts](https://docs.bentoml.org/en/latest/concepts/index.html) in BentoML +- Learn how to use BentoML with other ML Frameworks at [Frameworks Guide](https://docs.bentoml.org/en/latest/frameworks/index.html) or check out other [gallery projects](https://github.com/bentoml/BentoML/tree/main/examples) +- Learn more about model deployment options for Bento: + - [🦄️ Yatai](https://github.com/bentoml/Yatai): Model Deployment at scale on Kubernetes + - [🚀 bentoctl](https://github.com/bentoml/bentoctl): Fast model deployment on any cloud platform + diff --git a/examples/quickstart/bentofile.yaml b/examples/quickstart/bentofile.yaml new file mode 100644 index 0000000..9a4124c --- /dev/null +++ b/examples/quickstart/bentofile.yaml @@ -0,0 +1,10 @@ +service: "service.py:svc" +labels: + owner: bentoml-team + project: gallery +include: + - "*.py" +python: + packages: + - scikit-learn + - pandas diff --git a/examples/quickstart/iris_classifier.ipynb b/examples/quickstart/iris_classifier.ipynb new file mode 100644 index 0000000..faf5cb7 --- /dev/null +++ b/examples/quickstart/iris_classifier.ipynb @@ -0,0 +1,358 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "747e0e8d", + "metadata": {}, + "source": [ + "# BentoML Scikit-learn Tutorial\n", + "\n", + "\n", + "This is a sample project demonstrating basic usage of [BentoML](https://github.com/bentoml) with\n", + "Scikit-learn.\n", + "\n", + "In this project, we will train a classifier model using Scikit-learn and the Iris dataset, build\n", + "a prediction service for serving the trained model via an HTTP server, and containerize the \n", + "model server as a docker image for production deployment.\n", + "\n", + "\n", + "Link to source code: https://github.com/bentoml/BentoML/tree/main/examples/quickstart\n", + "\n", + "### Install Dependencies\n", + "\n", + "Install required python packages:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "daa3cbef", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -r https://raw.githubusercontent.com/bentoml/BentoML/main/examples/quickstart/requirements.txt" + ] + }, + { + "cell_type": "markdown", + "id": "b66e31f7", + "metadata": {}, + "source": [ + "## Model Training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb526488", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn import svm, datasets\n", + "\n", + "# Load training data\n", + "iris = datasets.load_iris()\n", + "X, y = iris.data, iris.target\n", + "\n", + "# Model Training\n", + "clf = svm.SVC()\n", + "clf.fit(X, y)" + ] + }, + { + "cell_type": "markdown", + "id": "3c114c75", + "metadata": {}, + "source": [ + "Save the `clf` model instance to BentoML local model store:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e94ed449", + "metadata": {}, + "outputs": [], + "source": [ + "import bentoml\n", + "\n", + "bentoml.sklearn.save_model(\"iris_clf\", clf)" + ] + }, + { + "cell_type": "markdown", + "id": "d613e57e", + "metadata": {}, + "source": [ + "Models saved can be accessed via `bentoml models` CLI command:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d771f04", + "metadata": {}, + "outputs": [], + "source": [ + "!bentoml models get iris_clf:latest" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a876780", + "metadata": {}, + "outputs": [], + "source": [ + "!bentoml models list" + ] + }, + { + "cell_type": "markdown", + "id": "672721c4", + "metadata": {}, + "source": [ + "To verify that the saved model can be loaded correctly:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28ac794b", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_model = bentoml.sklearn.load_model(\"iris_clf:latest\")\n", + "\n", + "loaded_model.predict([[5.9, 3.0, 5.1, 1.8]])" + ] + }, + { + "cell_type": "markdown", + "id": "8fd3bf97", + "metadata": {}, + "source": [ + "In BentoML, the recommended way of running ML model inference in serving is via Runner, which \n", + "gives BentoML more flexibility in terms of how to schedule the inference computation, how to \n", + "batch inference requests and take advantage of hardware resources available. Saved models can\n", + "be loaded as a Runner instance as shown below:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83205567", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a Runner instance:\n", + "iris_clf_runner = bentoml.sklearn.get(\"iris_clf:latest\").to_runner()\n", + "\n", + "# Runner#init_local initializes the model in current process, this is meant for development and testing only:\n", + "iris_clf_runner.init_local()\n", + "\n", + "# This should yield the same result as the loaded model:\n", + "iris_clf_runner.predict.run([[5.9, 3.0, 5.1, 1.8]])" + ] + }, + { + "cell_type": "markdown", + "id": "3fa68254", + "metadata": {}, + "source": [ + "## Serving the model\n", + "\n", + "A simple BentoML Service that serves the model saved above looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "127aa3fd", + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile service.py\n", + "import numpy as np\n", + "import bentoml\n", + "from bentoml.io import NumpyNdarray\n", + "\n", + "iris_clf_runner = bentoml.sklearn.get(\"iris_clf:latest\").to_runner()\n", + "\n", + "svc = bentoml.Service(\"iris_classifier\", runners=[iris_clf_runner])\n", + "\n", + "@svc.api(input=NumpyNdarray(), output=NumpyNdarray())\n", + "def classify(input_series: np.ndarray) -> np.ndarray:\n", + " return iris_clf_runner.predict.run(input_series)\n" + ] + }, + { + "cell_type": "markdown", + "id": "203beeed", + "metadata": {}, + "source": [ + "Note: using `%%writefile` here because `bentoml.Service` definition must be created in its own `.py` file\n", + "\n", + "Start a dev model server to test out the service defined above:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7523b58f", + "metadata": {}, + "outputs": [], + "source": [ + "!bentoml serve service.py:svc --reload" + ] + }, + { + "cell_type": "markdown", + "id": "3974e4ce", + "metadata": {}, + "source": [ + "\n", + "Open your web browser at http://127.0.0.1:3000 to view the Bento UI for sending test requests.\n", + "\n", + "You may also send request with `curl` command or any HTTP client, e.g.:\n", + "\n", + "```bash\n", + "curl -X POST -H \"content-type: application/json\" --data \"[[5.9, 3, 5.1, 1.8]]\" http://127.0.0.1:3000/classify\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "id": "4f1a8bcc", + "metadata": {}, + "source": [ + "### Build Bento for deployment" + ] + }, + { + "cell_type": "markdown", + "id": "d6192cd5", + "metadata": {}, + "source": [ + "Bento is the distribution format in BentoML which captures all the source code, model files, config\n", + "files and dependency specifications required for running the service for production deployment. Think \n", + "of it as Docker/Container designed for machine learning models.\n", + "\n", + "To begin with building Bento, create a `bentofile.yaml` under your project directory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6458e417", + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile bentofile.yaml\n", + "service: \"service.py:svc\"\n", + "labels:\n", + " owner: bentoml-team\n", + " project: gallery\n", + "include:\n", + "- \"*.py\"\n", + "python:\n", + " packages:\n", + " - scikit-learn\n", + " - pandas" + ] + }, + { + "cell_type": "markdown", + "id": "47505e3c", + "metadata": {}, + "source": [ + "Next, run `bentoml build` from current directory to start the Bento build:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7cab8b2", + "metadata": {}, + "outputs": [], + "source": [ + "!bentoml build" + ] + }, + { + "cell_type": "markdown", + "id": "4c159551", + "metadata": {}, + "source": [ + "A new Bento is now built and saved to local Bento store. You can view and manage it via \n", + "`bentoml list`,`bentoml get` and `bentoml delete` CLI command." + ] + }, + { + "cell_type": "markdown", + "id": "81ed8b84", + "metadata": {}, + "source": [ + "## Containerize and Deployment" + ] + }, + { + "cell_type": "markdown", + "id": "8c215454", + "metadata": {}, + "source": [ + "Bento is designed to be deployed to run efficiently in a variety of different environments.\n", + "And there are lots of deployment options and tools as part of the BentoML eco-system, such as \n", + "[Yatai](https://github.com/bentoml/Yatai) and [bentoctl](https://github.com/bentoml/bentoctl) for\n", + "direct deployment to cloud platforms.\n", + "\n", + "In this guide, we will show you the most basic way of deploying a Bento, which is converting a Bento\n", + "into a Docker image containing the HTTP model server.\n", + "\n", + "Make sure you have docker installed and docker deamon running, and run the following commnand:\n", + "\n", + "```bash\n", + "bentoml containerize iris_classifier:latest\n", + "```\n", + "\n", + "This will build a new docker image with all source code, model files and dependencies in place,\n", + "and ready for production deployment. To start a container with this docker image locally, run:\n", + "\n", + "```bash\n", + "docker run -p 3000:3000 iris_classifier:invwzzsw7li6zckb2ie5eubhd \n", + "```\n", + "\n", + "## What's Next?\n", + "\n", + "- 👉 [Pop into our Slack community!](https://l.linklyhq.com/l/ktO8) We're happy to help with any issue you face or even just to meet you and hear what you're working on.\n", + "\n", + "- Dive deeper into the [Core Concepts](https://docs.bentoml.org/en/latest/concepts/index.html) in BentoML\n", + "- Learn how to use BentoML with other ML Frameworks at [Frameworks Guide](https://docs.bentoml.org/en/latest/frameworks/index.html) or check out other [gallery projects](https://github.com/bentoml/BentoML/tree/main/examples)\n", + "- Learn more about model deployment options for Bento:\n", + " - [🦄️ Yatai](https://github.com/bentoml/Yatai): Model Deployment at scale on Kubernetes\n", + " - [🚀 bentoctl](https://github.com/bentoml/bentoctl): Fast model deployment on any cloud platform\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/quickstart/locustfile.py b/examples/quickstart/locustfile.py new file mode 100644 index 0000000..950e85a --- /dev/null +++ b/examples/quickstart/locustfile.py @@ -0,0 +1,111 @@ +import time + +import grpc +import numpy as np +from locust import task +from locust import User +from locust import between +from locust import HttpUser +from sklearn import datasets + +from bentoml.grpc.v1 import service_pb2 as pb +from bentoml.grpc.v1 import service_pb2_grpc as services + +test_data = datasets.load_iris().data +num_of_rows = test_data.shape[0] +max_batch_size = 10 + + +class IrisHttpUser(HttpUser): + """ + Usage: + Run the iris_classifier service in production mode: + + bentoml serve-http iris_classifier:latest --production + + Start locust load testing client with: + + locust --class-picker -H http://localhost:3000 + + Open browser at http://0.0.0.0:8089, adjust desired number of users and spawn + rate for the load test from the Web UI and start swarming. + """ + + @task + def classify(self): + start = np.random.choice(num_of_rows - max_batch_size) + end = start + np.random.choice(max_batch_size) + 1 + + input_data = test_data[start:end] + self.client.post("/classify", json=input_data.tolist()) + + wait_time = between(0.01, 2) + + +class GrpcUser(User): + abstract = True + + stub_class = None + + def __init__(self, environment): + super().__init__(environment) + self.environment = environment + + def on_start(self): + self.channel = grpc.insecure_channel(self.host) + self.stub = services.BentoServiceStub(self.channel) + + +class IrisGrpcUser(GrpcUser): + """ + Implementation is inspired by https://docs.locust.io/en/stable/testing-other-systems.html + + Usage: + Run the iris_classifier service in production mode: + + bentoml serve-grpc iris_classifier:latest --production + + Start locust load testing client with: + + locust --class-picker -H localhost:3000 + + Open browser at http://0.0.0.0:8089, adjust desired number of users and spawn + rate for the load test from the Web UI and start swarming. + """ + + @task + def classify(self): + start = np.random.choice(num_of_rows - max_batch_size) + end = start + np.random.choice(max_batch_size) + 1 + input_data = test_data[start:end] + request_meta = { + "request_type": "grpc", + "name": "classify", + "start_time": time.time(), + "response_length": 0, + "exception": None, + "context": None, + "response": None, + } + start_perf_counter = time.perf_counter() + try: + request_meta["response"] = self.stub.Call( + request=pb.Request( + api_name=request_meta["name"], + ndarray=pb.NDArray( + dtype=pb.NDArray.DTYPE_FLOAT, + # shape=(1, 4), + shape=(len(input_data), 4), + # float_values=[5.9, 3, 5.1, 1.8], + float_values=input_data.flatten(), + ), + ) + ) + except grpc.RpcError as e: + request_meta["exception"] = e + request_meta["response_time"] = ( + time.perf_counter() - start_perf_counter + ) * 1000 + self.environment.events.request.fire(**request_meta) + + wait_time = between(0.01, 2) diff --git a/examples/quickstart/query.py b/examples/quickstart/query.py new file mode 100644 index 0000000..cf9ef0d --- /dev/null +++ b/examples/quickstart/query.py @@ -0,0 +1,9 @@ +import requests + +result = requests.post( + "http://127.0.0.1:3000/classify", + headers={"content-type": "application/json"}, + data="[[5.9, 3, 5.1, 1.8]]", +).text + +print(result) \ No newline at end of file diff --git a/examples/quickstart/query.sh b/examples/quickstart/query.sh new file mode 100644 index 0000000..aa9240a --- /dev/null +++ b/examples/quickstart/query.sh @@ -0,0 +1,4 @@ +curl -X POST \ + -H "content-type: application/json" \ + --data "[[5.9, 3, 5.1, 1.8]]" \ + http://127.0.0.1:3000/classify \ No newline at end of file diff --git a/examples/quickstart/requirements.txt b/examples/quickstart/requirements.txt new file mode 100644 index 0000000..98ac078 --- /dev/null +++ b/examples/quickstart/requirements.txt @@ -0,0 +1,3 @@ +scikit-learn +pandas +bentoml>=1.0.0 diff --git a/examples/quickstart/service.py b/examples/quickstart/service.py new file mode 100644 index 0000000..86b1562 --- /dev/null +++ b/examples/quickstart/service.py @@ -0,0 +1,18 @@ +import numpy as np + +import bentoml +from bentoml.io import NumpyNdarray + +iris_clf_runner = bentoml.sklearn.get("iris_clf:latest").to_runner() + +svc = bentoml.Service("iris_classifier", runners=[iris_clf_runner]) + + +@svc.api( + input=NumpyNdarray.from_sample( + np.array([[4.9, 3.0, 1.4, 0.2]], dtype=np.double), enforce_shape=False + ), + output=NumpyNdarray(), +) +async def classify(input_series: np.ndarray) -> np.ndarray: + return await iris_clf_runner.predict.async_run(input_series) diff --git a/examples/quickstart/train.py b/examples/quickstart/train.py new file mode 100644 index 0000000..6ebf990 --- /dev/null +++ b/examples/quickstart/train.py @@ -0,0 +1,24 @@ +import logging + +from sklearn import svm +from sklearn import datasets + +import bentoml + +logging.basicConfig(level=logging.WARN) + +if __name__ == "__main__": + + # Load training data + iris = datasets.load_iris() + X, y = iris.data, iris.target + + # Model Training + clf = svm.SVC() + clf.fit(X, y) + + # Save model to BentoML local model store + saved_model = bentoml.sklearn.save_model( + "iris_clf", clf, signatures={"predict": {"batchable": True, "batch_dim": 0}} + ) + print(f"Model saved: {saved_model}") diff --git a/four-score.m4a.srt b/four-score.m4a.srt new file mode 100644 index 0000000..2f70a94 --- /dev/null +++ b/four-score.m4a.srt @@ -0,0 +1,32 @@ +1 +00:00:00,000 --> 00:00:08,120 +Four score and seven years ago our fathers brought forth on this continent a new nation + +2 +00:00:08,120 --> 00:00:14,800 +conceived in liberty and dedicated to the proposition that all men are created equal. + +3 +00:00:14,800 --> 00:00:21,000 +Now we are engaged in a great civil war testing whether that nation or any nation so conceived + +4 +00:00:21,000 --> 00:00:24,220 +and so dedicated can long endure. + +5 +00:00:24,220 --> 00:00:26,720 +We are met on a great battlefield of that war. + +6 +00:00:26,720 --> 00:00:33,100 +We have come to dedicate a portion of that field as a final resting place for those here + +7 +00:00:33,100 --> 00:00:36,060 +gave their lives that the nation might live. + +8 +00:00:36,060 --> 00:00:52,340 +It is altogether fitting and proper that we should do this. + diff --git a/four-score.m4a.txt b/four-score.m4a.txt new file mode 100644 index 0000000..6e0bc8f --- /dev/null +++ b/four-score.m4a.txt @@ -0,0 +1,8 @@ +Four score and seven years ago our fathers brought forth on this continent a new nation +conceived in liberty and dedicated to the proposition that all men are created equal. +Now we are engaged in a great civil war testing whether that nation or any nation so conceived +and so dedicated can long endure. +We are met on a great battlefield of that war. +We have come to dedicate a portion of that field as a final resting place for those here +gave their lives that the nation might live. +It is altogether fitting and proper that we should do this. diff --git a/four-score.m4a.vtt b/four-score.m4a.vtt new file mode 100644 index 0000000..f9baf56 --- /dev/null +++ b/four-score.m4a.vtt @@ -0,0 +1,26 @@ +WEBVTT + +00:00.000 --> 00:08.120 +Four score and seven years ago our fathers brought forth on this continent a new nation + +00:08.120 --> 00:14.800 +conceived in liberty and dedicated to the proposition that all men are created equal. + +00:14.800 --> 00:21.000 +Now we are engaged in a great civil war testing whether that nation or any nation so conceived + +00:21.000 --> 00:24.220 +and so dedicated can long endure. + +00:24.220 --> 00:26.720 +We are met on a great battlefield of that war. + +00:26.720 --> 00:33.100 +We have come to dedicate a portion of that field as a final resting place for those here + +00:33.100 --> 00:36.060 +gave their lives that the nation might live. + +00:36.060 --> 00:52.340 +It is altogether fitting and proper that we should do this. + diff --git a/hugging-face/download_hf_model.py b/hugging-face/download_hf_model.py new file mode 100644 index 0000000..6364d66 --- /dev/null +++ b/hugging-face/download_hf_model.py @@ -0,0 +1,13 @@ +""" +After running zip it to save +zip -r summarizeApp.zip summarizeApp +""" + +from transformers import pipeline + +model = pipeline( + "summarization", + model="sshleifer/distilbart-cnn-12-6", + revision="a4f8f3e", +) +model.save_pretrained("summarizeApp") \ No newline at end of file diff --git a/hugging-face/hf_fine_tune_hello_world.py b/hugging-face/hf_fine_tune_hello_world.py new file mode 100644 index 0000000..c370e24 --- /dev/null +++ b/hugging-face/hf_fine_tune_hello_world.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +""" +Fine Tuning Example with HuggingFace + +Based on official tutorial +""" + +from transformers import AutoTokenizer +from datasets import load_dataset, load_metric +from transformers import AutoModelForSequenceClassification +from transformers import TrainingArguments, Trainer +import numpy as np + +# Load the dataset +dataset = load_dataset("yelp_review_full") +dataset["train"][100] +tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + +def tokenize_function(examples): + return tokenizer(examples["text"], padding="max_length", truncation=True) +tokenized_datasets = dataset.map(tokenize_function, batched=True) + +# Load the model +model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5) + +metric = load_metric("accuracy") +def compute_metrics(eval_pred): + logits, labels = eval_pred + predictions = np.argmax(logits, axis=-1) + return metric.compute(predictions=predictions, references=labels) + +# can use if needed to reduce memory usage and training time +small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000)) +small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000)) + + +training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch") +trainer = Trainer( + model=model, + args=training_args, + train_dataset=small_train_dataset, + eval_dataset=small_eval_dataset, + compute_metrics=compute_metrics, +) + +trainer.train() # train the model \ No newline at end of file diff --git a/hugging-face/hf_whisper.py b/hugging-face/hf_whisper.py new file mode 100755 index 0000000..954ab0e --- /dev/null +++ b/hugging-face/hf_whisper.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# """Create OpenAI Whisper command-line tool using Hugging Face's transformers library.""" + +from transformers import pipeline +import click + + +# Create a function that reads a sample audio file and transcribes it using openai's whisper +def traudio(filename, model="openai/whisper-tiny.en"): + with open(filename, "rb") as f: + _ = f.read() # this needs to be fixed + print(f"Transcribing {filename}...") + pipe = pipeline("automatic-speech-recognition", model=model) + results = pipe(filename) + return results + + +# create click group +@click.group() +def cli(): + """A cli for openai whisper""" + + +# create a click command that transcribes +@cli.command("transcribe") +@click.option( + "--model", default="openai/whisper-tiny.en", help="Model to use for transcription" +) +@click.argument("filename", default="utils/four-score.m4a") +def whispercli(filename, model): + """Transcribe audio using openai whisper""" + results = traudio(filename, model) + # print out each label and its score in a tabular format with colors + for result in results: + click.secho(f"{result['text']}", fg="green") + + +if __name__ == "__main__": + cli() diff --git a/hugging-face/load_model.py b/hugging-face/load_model.py new file mode 100644 index 0000000..6bc3b3b --- /dev/null +++ b/hugging-face/load_model.py @@ -0,0 +1,14 @@ +"""Loading a model from a file.""" + +from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + +model = AutoModelForSeq2SeqLM.from_pretrained("summarizeApp") +tokenizer = AutoTokenizer.from_pretrained("summarizeApp") +#open the file with utf-8 encoding +with open("input.txt", encoding="utf-8") as f: + text = f.read() + +input_ids = tokenizer.encode(text, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) \ No newline at end of file diff --git a/hugging-face/zero_shot_classification.py b/hugging-face/zero_shot_classification.py new file mode 100755 index 0000000..1407807 --- /dev/null +++ b/hugging-face/zero_shot_classification.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +"""Create Zero-shot classification command-line tool using Hugging Face's transformers library.""" + +from transformers import pipeline +import click + +# Create a function that reads a file +def read_file(filename): + with open(filename, encoding="utf-8") as myfile: + return myfile.read() + + +# create a function that grabs candidate labels from a file +def read_labels(kw_file): + return read_file(kw_file).splitlines() + + +# create a function that reads a file performs zero-shot classification +def classify(text, labels, model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli"): + classifier = pipeline("zero-shot-classification", model=model) + results = classifier(text, labels, multi_label=False) + return results + + +# create click group +@click.group() +def cli(): + """A cli for zero-shot classification""" + + +# create a click command that performs zero-shot classification +@cli.command("classify") +@click.argument("filename", default="four-score.m4a.txt") +@click.argument("kw_file", default="keywords.txt") +def classifycli(filename, kw_file): + """Classify text using zero-shot classification""" + text = read_file(filename) + labels = read_labels(kw_file) # needs to be a sequence + results = classify(text, labels) + # print out each label and its score in a tabular format with colors + for label, score in zip(results["labels"], results["scores"]): + click.secho(f"{label}\t{score:.2f}", fg="green") + + +if __name__ == "__main__": + cli() diff --git a/input.txt b/input.txt new file mode 100644 index 0000000..5d6a565 --- /dev/null +++ b/input.txt @@ -0,0 +1,8 @@ +Fourscore and seven years ago our fathers brought forth on this continent a new nation +conceived in liberty and dedicated to the proposition that all men are created equal. +Now we are engaged in a great civil war testing whether that nation or any nation so conceived +and so dedicated can long endure. +We are met on a great battlefield of that war. +We have come to dedicate a portion of that field as a final resting place for those here +gave their lives that the nation might live. +It is all together fitting and proper that we should do this. diff --git a/keywords.txt b/keywords.txt new file mode 100644 index 0000000..79c8dce --- /dev/null +++ b/keywords.txt @@ -0,0 +1,5 @@ +created equal +years ago +ago our fathers +fathers brought +men are created diff --git a/main.py b/main.py new file mode 100644 index 0000000..0af0692 --- /dev/null +++ b/main.py @@ -0,0 +1,18 @@ +""" +Main cli or app entry point +""" + +from mylib.calculator import add +import click + + +@click.command("add") +@click.argument("a", type=int) +@click.argument("b", type=int) +def add_cli(a, b): + click.echo(add(a, b)) + + +if __name__ == "__main__": + # pylint: disable=no-value-for-parameter + add_cli() diff --git a/mylib/__init__.py b/mylib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mylib/calculator.py b/mylib/calculator.py new file mode 100644 index 0000000..754cf26 --- /dev/null +++ b/mylib/calculator.py @@ -0,0 +1,19 @@ +""" +Calculations library +""" + + +def add(a, b): + return a + b + + +def subtract(a, b): + return a - b + + +def multiply(a, b): + return a * b + + +def divide(a, b): + return a / b \ No newline at end of file diff --git a/repeat.sh b/repeat.sh new file mode 100644 index 0000000..d91ad3b --- /dev/null +++ b/repeat.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +repeat() { + for i in `seq 1 $1`; do + echo $2 + done +} + +repeat $1 $2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d7986b5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,46 @@ +#wheel +wheel==0.37.1 +#devops +black==22.3.0 +click==8.1.3 +pytest==7.1.3 +pytest-cov==4.0.0 +pylint==2.15.3 +boto3==1.24.87 +#web +fastapi==0.85.0 +uvicorn==0.18.3 +#datascience +jupyter==1.0.0 +pandas==1.5.0 +numpy==1.23.3 +scikit-learn==1.1.2 +matplotlib==3.6.0 +seaborn==0.12.0 +dask==2022.9.2 +#geo +geopy==2.2.0 +#MLOps +gradio==3.4.1 +streamlit==1.13.0 +protobuf==3.16.0 +onnx==1.12.0 +databricks==0.2 +databricks-cli==0.17.3 +databricks-sql-connector==2.1.0 +mlflow==2.0.1 +##MLOps-HuggingFace +transformers==4.22.2 +sentencepiece==0.1.97 +datasets==2.5.2 +tokenizers==0.12.1 +#deep learning +torch +torchvision +tensorflow +#openai whisper +git+https://github.com/openai/whisper.git +#yake +yake==0.4.8 + + diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..f6f2b6c --- /dev/null +++ b/setup.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +source /home/codespace/venv/bin/activate +#append it to bash so every shell launches with it +echo 'source /home/codespace/venv/bin/activate' >> ~/.bashrc diff --git a/test_main.py b/test_main.py new file mode 100644 index 0000000..92cde4d --- /dev/null +++ b/test_main.py @@ -0,0 +1,10 @@ +""" +Test goes here + +""" + +from mylib.calculator import add + + +def test_add(): + assert add(1, 2) == 3 diff --git a/tf-requirements.txt b/tf-requirements.txt new file mode 100644 index 0000000..052f995 --- /dev/null +++ b/tf-requirements.txt @@ -0,0 +1,2 @@ +#Tensorflow requirements +tensorflow==2.10.0 \ No newline at end of file diff --git a/utils/README.md b/utils/README.md new file mode 100644 index 0000000..d3ca112 --- /dev/null +++ b/utils/README.md @@ -0,0 +1,7 @@ +Using CPP Whisper: https://github.com/ggerganov/whisper.cpp/issues/89 +ffmpeg -i four-score.m4a -ac 2 -ar 16000 -f wav four-score.wav +#time ./main -f ../mlops-template/utils/four-score.wav +time ./main -m models/ggml-medium.en.bin -f ../mlops-template/utils/four-score.wav +##fastest test +time ./main -m models/ggml-medium.en.bin -f ../mlops-template/utils/four-score.wav -t 8 -p 2 + diff --git a/utils/four-score.m4a b/utils/four-score.m4a new file mode 100644 index 0000000..db827ce Binary files /dev/null and b/utils/four-score.m4a differ diff --git a/utils/four-score.wav b/utils/four-score.wav new file mode 100644 index 0000000..70646a5 Binary files /dev/null and b/utils/four-score.wav differ diff --git a/utils/install-nvidia-container-toolkit.sh b/utils/install-nvidia-container-toolkit.sh new file mode 100755 index 0000000..f75a272 --- /dev/null +++ b/utils/install-nvidia-container-toolkit.sh @@ -0,0 +1,14 @@ +#installs NVIDIA container Toolkit +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#install-guide +distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ + && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + +sudo apt-get update +sudo apt-get install -y nvidia-docker2 +sudo systemctl restart docker + +#then start +#sudo docker run --rm --gpus all nvidia/cuda:11.6.2-base-ubuntu20.04 nvidia-smi \ No newline at end of file diff --git a/utils/kw_extract.py b/utils/kw_extract.py new file mode 100755 index 0000000..33eceb7 --- /dev/null +++ b/utils/kw_extract.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +""" +Main cli or app entry point +""" +import yake +import click + +# create a function that reads a file +def read_file(filename): + with open(filename, encoding="utf-8") as myfile: + return myfile.read() + + +# def extract keywords +def extract_keywords(text): + kw_extractor = yake.KeywordExtractor() + keywords = kw_extractor.extract_keywords(text) + return keywords + + +# create a function that makes hash tags +def make_hashtags(keywords): + hashtags = [] + for keyword in keywords: + hashtags.append("#" + keyword[0].replace(" ", "")) + return hashtags + +#create a function returns only the top N keywords +def top_n_keywords(keywords, n=5): + return keywords[:n] + +@click.group() +def cli(): + """A cli for keyword extraction""" + +#print the top N keywords +@cli.command("extract") +@click.argument("filename", default="text.txt") +@click.option("--n", default=5, help="Number of keywords to extract") +def extract(filename, n): + """Extract keywords from a file""" + text = read_file(filename) + keywords = extract_keywords(text) + top_keywords = top_n_keywords(keywords, n) + for keyword in top_keywords: + print(keyword[0]) + + +@cli.command("hashtags") +@click.argument("filename", default="text.txt") +def hashtagscli(filename): + """Extract keywords from a file and make hashtags""" + text = read_file(filename) + keywords = extract_keywords(text) + hashtags = make_hashtags(keywords) + click.echo(hashtags) + + +if __name__ == "__main__": + cli() \ No newline at end of file diff --git a/utils/quickstart_pytorch.py b/utils/quickstart_pytorch.py new file mode 100644 index 0000000..7289b1c --- /dev/null +++ b/utils/quickstart_pytorch.py @@ -0,0 +1,241 @@ +""" +`Learn the Basics `_ || +**Quickstart** || +`Tensors `_ || +`Datasets & DataLoaders `_ || +`Transforms `_ || +`Build Model `_ || +`Autograd `_ || +`Optimization `_ || +`Save & Load Model `_ + +Quickstart +=================== +This section runs through the API for common tasks in machine learning. Refer to the links in each section to dive deeper. + +Working with data +----------------- +PyTorch has two `primitives to work with data `_: +``torch.utils.data.DataLoader`` and ``torch.utils.data.Dataset``. +``Dataset`` stores the samples and their corresponding labels, and ``DataLoader`` wraps an iterable around +the ``Dataset``. + +""" + +import torch +from torch import nn +from torch.utils.data import DataLoader +from torchvision import datasets +from torchvision.transforms import ToTensor + +###################################################################### +# PyTorch offers domain-specific libraries such as `TorchText `_, +# `TorchVision `_, and `TorchAudio `_, +# all of which include datasets. For this tutorial, we will be using a TorchVision dataset. +# +# The ``torchvision.datasets`` module contains ``Dataset`` objects for many real-world vision data like +# CIFAR, COCO (`full list here `_). In this tutorial, we +# use the FashionMNIST dataset. Every TorchVision ``Dataset`` includes two arguments: ``transform`` and +# ``target_transform`` to modify the samples and labels respectively. + +# Download training data from open datasets. +training_data = datasets.FashionMNIST( + root="data", + train=True, + download=True, + transform=ToTensor(), +) + +# Download test data from open datasets. +test_data = datasets.FashionMNIST( + root="data", + train=False, + download=True, + transform=ToTensor(), +) + +###################################################################### +# We pass the ``Dataset`` as an argument to ``DataLoader``. This wraps an iterable over our dataset, and supports +# automatic batching, sampling, shuffling and multiprocess data loading. Here we define a batch size of 64, i.e. each element +# in the dataloader iterable will return a batch of 64 features and labels. + +batch_size = 64 + +# Create data loaders. +train_dataloader = DataLoader(training_data, batch_size=batch_size) +test_dataloader = DataLoader(test_data, batch_size=batch_size) + +for X, y in test_dataloader: + print(f"Shape of X [N, C, H, W]: {X.shape}") + print(f"Shape of y: {y.shape} {y.dtype}") + break + +###################################################################### +# Read more about `loading data in PyTorch `_. +# + +###################################################################### +# -------------- +# + +################################ +# Creating Models +# ------------------ +# To define a neural network in PyTorch, we create a class that inherits +# from `nn.Module `_. We define the layers of the network +# in the ``__init__`` function and specify how data will pass through the network in the ``forward`` function. To accelerate +# operations in the neural network, we move it to the GPU if available. + +# Get cpu or gpu device for training. +device = "cuda" if torch.cuda.is_available() else "cpu" +print(f"Using {device} device") + +# Define model +class NeuralNetwork(nn.Module): + def __init__(self): + super().__init__() + self.flatten = nn.Flatten() + self.linear_relu_stack = nn.Sequential( + nn.Linear(28*28, 512), + nn.ReLU(), + nn.Linear(512, 512), + nn.ReLU(), + nn.Linear(512, 10) + ) + + def forward(self, x): + x = self.flatten(x) + logits = self.linear_relu_stack(x) + return logits + +model = NeuralNetwork().to(device) +print(model) + +###################################################################### +# Read more about `building neural networks in PyTorch `_. +# + + +###################################################################### +# -------------- +# + + +##################################################################### +# Optimizing the Model Parameters +# ---------------------------------------- +# To train a model, we need a `loss function `_ +# and an `optimizer `_. + +loss_fn = nn.CrossEntropyLoss() +optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + + +####################################################################### +# In a single training loop, the model makes predictions on the training dataset (fed to it in batches), and +# backpropagates the prediction error to adjust the model's parameters. + +def train(dataloader, model, loss_fn, optimizer): + size = len(dataloader.dataset) + model.train() + for batch, (X, y) in enumerate(dataloader): + X, y = X.to(device), y.to(device) + + # Compute prediction error + pred = model(X) + loss = loss_fn(pred, y) + + # Backpropagation + optimizer.zero_grad() + loss.backward() + optimizer.step() + + if batch % 100 == 0: + loss, current = loss.item(), batch * len(X) + print(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") + +############################################################################## +# We also check the model's performance against the test dataset to ensure it is learning. + +def test(dataloader, model, loss_fn): + size = len(dataloader.dataset) + num_batches = len(dataloader) + model.eval() + test_loss, correct = 0, 0 + with torch.no_grad(): + for X, y in dataloader: + X, y = X.to(device), y.to(device) + pred = model(X) + test_loss += loss_fn(pred, y).item() + correct += (pred.argmax(1) == y).type(torch.float).sum().item() + test_loss /= num_batches + correct /= size + print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + +############################################################################## +# The training process is conducted over several iterations (*epochs*). During each epoch, the model learns +# parameters to make better predictions. We print the model's accuracy and loss at each epoch; we'd like to see the +# accuracy increase and the loss decrease with every epoch. + +epochs = 5 +for t in range(epochs): + print(f"Epoch {t+1}\n-------------------------------") + train(train_dataloader, model, loss_fn, optimizer) + test(test_dataloader, model, loss_fn) +print("Done!") + +###################################################################### +# Read more about `Training your model `_. +# + +###################################################################### +# -------------- +# + +###################################################################### +# Saving Models +# ------------- +# A common way to save a model is to serialize the internal state dictionary (containing the model parameters). + +torch.save(model.state_dict(), "model.pth") +print("Saved PyTorch Model State to model.pth") + + + +###################################################################### +# Loading Models +# ---------------------------- +# +# The process for loading a model includes re-creating the model structure and loading +# the state dictionary into it. + +model = NeuralNetwork() +model.load_state_dict(torch.load("model.pth")) + +############################################################# +# This model can now be used to make predictions. + +classes = [ + "T-shirt/top", + "Trouser", + "Pullover", + "Dress", + "Coat", + "Sandal", + "Shirt", + "Sneaker", + "Bag", + "Ankle boot", +] + +model.eval() +x, y = test_data[0][0], test_data[0][1] +with torch.no_grad(): + pred = model(x) + predicted, actual = classes[pred[0].argmax(0)], classes[y] + print(f'Predicted: "{predicted}", Actual: "{actual}"') + + +###################################################################### +# Read more about `Saving & Loading your model `_. +# \ No newline at end of file diff --git a/utils/quickstart_tf2.py b/utils/quickstart_tf2.py new file mode 100644 index 0000000..9ebe84b --- /dev/null +++ b/utils/quickstart_tf2.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import tensorflow as tf +print("TensorFlow version:", tf.__version__) + +mnist = tf.keras.datasets.mnist + +(x_train, y_train), (x_test, y_test) = mnist.load_data() +x_train, x_test = x_train / 255.0, x_test / 255.0 +model = tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128, activation='relu'), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) +]) +predictions = model(x_train[:1]).numpy() +predictions +tf.nn.softmax(predictions).numpy() +loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) +loss_fn(y_train[:1], predictions).numpy() +model.compile(optimizer='adam', + loss=loss_fn, + metrics=['accuracy']) +model.fit(x_train, y_train, epochs=5) +model.evaluate(x_test, y_test, verbose=2) +probability_model = tf.keras.Sequential([ + model, + tf.keras.layers.Softmax() +]) +probability_model(x_test[:5]) \ No newline at end of file diff --git a/utils/transcribe-whisper.sh b/utils/transcribe-whisper.sh new file mode 100755 index 0000000..07bf87f --- /dev/null +++ b/utils/transcribe-whisper.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +whisper utils/four-score.m4a --model large --language English \ No newline at end of file diff --git a/utils/verify_pytorch.py b/utils/verify_pytorch.py new file mode 100644 index 0000000..9986aa1 --- /dev/null +++ b/utils/verify_pytorch.py @@ -0,0 +1,12 @@ +import torch + +if torch.cuda.is_available(): + print("CUDA is available") + print("CUDA version: {}".format(torch.version.cuda)) + print("PyTorch version: {}".format(torch.__version__)) + print("cuDNN version: {}".format(torch.backends.cudnn.version())) + print("Number of CUDA devices: {}".format(torch.cuda.device_count())) + print("Current CUDA device: {}".format(torch.cuda.current_device())) + print("Device name: {}".format(torch.cuda.get_device_name(torch.cuda.current_device()))) +else: + print("CUDA is not available") \ No newline at end of file diff --git a/utils/verify_tf.py b/utils/verify_tf.py new file mode 100644 index 0000000..c1a53cb --- /dev/null +++ b/utils/verify_tf.py @@ -0,0 +1 @@ +import tensorflow as tf; print(tf.config.list_physical_devices('GPU')) \ No newline at end of file