Add example-demo and documentation.

latitudegames · Jun 13, 2023 · eef9f79 · eef9f79
1 parent cc3c16f
commit eef9f79
Show file tree

Hide file tree

Showing 33 changed files with 30,429 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+example-demo/node_modules/
diff --git a/.npmignore b/.npmignore
@@ -0,0 +1 @@
+example-demo/
diff --git a/README.md b/README.md
@@ -4,6 +4,8 @@ The first JavaScript tokenizer for LLaMA which works client-side in the browser
 
 Intended use case is calculating token count accurately on the client-side.
 
+<a href="https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/">Click here for example-demo</a>
+
 Features:
 - Easy to use: 0 dependencies, code and data baked into a single file.
 - Compatible with most LLaMA-based models (see [Compatibility](#compatibility))
@@ -27,7 +29,7 @@ console.log(llamaTokenizer.encode("Hello world!").length)
 Option 2: Load as ES6 module with `<script>` tags in your HTML
 
 ```
-<script type="module" src="https://raw.githubusercontent.com/belladoreai/llama-tokenizer-js/master/llama-tokenizer.js"></script>
+<script type="module" src="https://belladoreai.github.io/llama-tokenizer-js/llama-tokenizer.js"></script>
 ```
 
 ## Usage
@@ -39,7 +41,7 @@ When used in browser, llama-tokenizer-js pollutes global namespace with `llamaTo
 Encode:
 
 ```
-llamaTokenizer.encode("Hello world")
+llamaTokenizer.encode("Hello world!")
 > [1, 15043, 3186, 29991]
 ```
 
@@ -83,4 +85,4 @@ Incompatible models are those which have been trained from scratch, not on top o
 
 You are free to use llama-tokenizer-js for basically whatever you want (MIT license).
 
-You are not required to give anything in exchange, but I kindly ask that you link to [https://belladore.ai/tools](https://belladore.ai/tools) in an appropriate place in your website. For example, you might link with the text "Using llama-tokenizer-js by belladore.ai" or something similar.
+You are not required to give anything in exchange, but I kindly ask that you give back by linking to [https://belladore.ai/tools](https://belladore.ai/tools) in an appropriate place in your website. For example, you might link with the text "Using llama-tokenizer-js by belladore.ai" or something similar.
diff --git a/example-demo/.gitignore b/example-demo/.gitignore
@@ -0,0 +1,20 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
diff --git a/example-demo/README.md b/example-demo/README.md
@@ -0,0 +1,10 @@
+# Getting Started
+
+This is a playground for llama-tokenizer-js
+
+This playground was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
+
+```
+npm install
+npm start
+```
diff --git a/example-demo/build/asset-manifest.json b/example-demo/build/asset-manifest.json
@@ -0,0 +1,13 @@
+{
+  "files": {
+    "main.css": "/static/css/main.82f7171e.css",
+    "main.js": "/static/js/main.70938086.js",
+    "index.html": "/index.html",
+    "main.82f7171e.css.map": "/static/css/main.82f7171e.css.map",
+    "main.70938086.js.map": "/static/js/main.70938086.js.map"
+  },
+  "entrypoints": [
+    "static/css/main.82f7171e.css",
+    "static/js/main.70938086.js"
+  ]
+}
diff --git a/example-demo/build/favicon.ico b/example-demo/build/favicon.ico
diff --git a/example-demo/build/github.png b/example-demo/build/github.png
diff --git a/example-demo/build/index.html b/example-demo/build/index.html
@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>llama-tokenizer-js playground</title><script defer="defer" src="/static/js/main.70938086.js"></script><link href="/static/css/main.82f7171e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
diff --git a/example-demo/build/logo192.png b/example-demo/build/logo192.png
diff --git a/example-demo/build/logo512.png b/example-demo/build/logo512.png
diff --git a/example-demo/build/manifest.json b/example-demo/build/manifest.json
@@ -0,0 +1,25 @@
+{
+  "short_name": "React App",
+  "name": "Create React App Sample",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    },
+    {
+      "src": "logo192.png",
+      "type": "image/png",
+      "sizes": "192x192"
+    },
+    {
+      "src": "logo512.png",
+      "type": "image/png",
+      "sizes": "512x512"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}
diff --git a/example-demo/build/npm.png b/example-demo/build/npm.png
diff --git a/example-demo/build/robots.txt b/example-demo/build/robots.txt
@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:
diff --git a/example-demo/build/static/css/main.82f7171e.css b/example-demo/build/static/css/main.82f7171e.css
diff --git a/example-demo/build/static/css/main.82f7171e.css.map b/example-demo/build/static/css/main.82f7171e.css.map
diff --git a/example-demo/build/static/js/main.70938086.js b/example-demo/build/static/js/main.70938086.js
diff --git a/example-demo/build/static/js/main.70938086.js.LICENSE.txt b/example-demo/build/static/js/main.70938086.js.LICENSE.txt
@@ -0,0 +1,39 @@
+/**
+ * @license React
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
diff --git a/example-demo/build/static/js/main.70938086.js.map b/example-demo/build/static/js/main.70938086.js.map
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>llama-tokenizer-js playground</title><script defer="defer" src="/static/js/main.70938086.js"></script><link href="/static/css/main.82f7171e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>