feat(paddlejs-model): add detect model

JingyuanZhang · Feb 14, 2022 · 58e5e78 · 58e5e78
1 parent 84334f0
commit 58e5e78
Show file tree

Hide file tree

Showing 17 changed files with 1,949 additions and 4 deletions.
diff --git a/e2e/dist/assets/detect.json b/e2e/dist/assets/detect.json
@@ -0,0 +1,42 @@
+[
+    [
+        1,
+        0.9999813437461853,
+        0.3285195231437683,
+        0.6718374490737915,
+        0.4688636064529419,
+        0.8186416625976562
+    ],
+    [
+        1,
+        0.9999707937240601,
+        0.2805149555206299,
+        0.48382753133773804,
+        0.42529064416885376,
+        0.635986864566803
+    ],
+    [
+        1,
+        0.9983498454093933,
+        0.5017392039299011,
+        0.5820830464363098,
+        0.5703544020652771,
+        0.8374423384666443
+    ],
+    [
+        1,
+        0.9950063824653625,
+        0.4869859516620636,
+        0.23868629336357117,
+        0.5620469450950623,
+        0.4887562096118927
+    ],
+    [
+        2,
+        0.9992570281028748,
+        0.36442363262176514,
+        0.4332561790943146,
+        0.43267953395843506,
+        0.5595301389694214
+    ]
+]
diff --git a/e2e/dist/assets/imgs/detect.jpeg b/e2e/dist/assets/imgs/detect.jpeg
diff --git a/e2e/dist/index.html b/e2e/dist/index.html
@@ -18,14 +18,16 @@
                 height: 640px;
             }
         </style>
+
     </head>
     <body>
         <img id="car" src="./assets/imgs/car.webp"/>
         <img id="banana" src="./assets/imgs/banana.jpeg"/>
         <img id="ok" src="./assets/imgs/ok.jpeg"/>
         <img id="human" src="./assets/imgs/human.jpg"/>
         <img id="seg" src="./assets/imgs/seg.png"/>
-        <img id="ocr" src="./assets/imgs/ocr.jpg" />
+        <img id="ocr" src="./assets/imgs/ocr.jpg"/>
+        <img id="detect" src="./assets/imgs/detect.jpeg"/>
         <canvas id="back_canvas"></canvas>
         <canvas id="seg_canvas"></canvas>
     </body>
@@ -35,4 +37,5 @@
     <script src="./gesture_bundle.js"></script>
     <script src="./humanseg_bundle.js"></script>
     <script src="./ocr_bundle.js"></script>
+    <script src="./detect_bundle.js"></script>
 </html>
diff --git a/e2e/server.ts b/e2e/server.ts
@@ -8,6 +8,7 @@ const mobilenetWebpackConfig = require('../packages/paddlejs-models/mobilenet/we
 const gestureWebpackConfig = require('../packages/paddlejs-models/gesture/webpack.prod');
 const humansegWebpackConfig = require('../packages/paddlejs-models/humanseg/webpack.prod');
 const ocrWebpackConfig = require('../packages/paddlejs-models/ocr/webpack.prod');
+const detectWebpackConfig = require('../packages/paddlejs-models/detect/webpack.prod');
 
 const DIST_DIR = path.join(__dirname, 'dist');
 
@@ -33,9 +34,10 @@ const mobilenet = new ConfigInfo('mobilenet', mobilenetWebpackConfig, true);
 const gesture = new ConfigInfo('gesture', gestureWebpackConfig, true);
 const humanseg = new ConfigInfo('humanseg', humansegWebpackConfig, true);
 const ocr = new ConfigInfo('ocr', ocrWebpackConfig, true);
+const detect = new ConfigInfo('detect', detectWebpackConfig, true);
 
 // edit webpack config
-[core, webgl, mobilenet, gesture, humanseg, ocr].forEach(instance => {
+[core, webgl, mobilenet, gesture, humanseg, ocr, detect].forEach(instance => {
     const config = instance.config;
     config.output.path = DIST_DIR;
     config.output.filename = `${instance.key}_bundle.js`;
@@ -64,6 +66,7 @@ const app = express()
     .use(middleware(gesture.compiler))
     .use(middleware(humanseg.compiler))
     .use(middleware(ocr.compiler))
+    .use(middleware(detect.compiler))
     .use(express.static(DIST_DIR));
 
 app.listen(port, () => {

diff --git a/e2e/tests/detect.test.ts b/e2e/tests/detect.test.ts
@@ -0,0 +1,38 @@
+const detectResult = require('../dist/assets/detect.json');
+
+describe('e2e test detect model', () => {
+    const { paddlejs } = require('./global.d.ts');
+    const CUR_URL = 'http://localhost:9898/';
+
+    beforeAll(async () => {
+        await page.goto(CUR_URL);
+    });
+
+    it('detect predict', async () => {
+        page.on('console', msg => console.log('PAGE LOG:', msg.text()));
+        const res = await page.evaluate(async () => {
+            const $detect = document.querySelector('#detect');
+            const det = paddlejs['detect'];
+            await det.init();
+            const result = await det.detect($detect);
+            return result;
+        });
+        const gap = 0.02;
+
+        detectResult.forEach((item, index) => {
+            // label 对比，若结果与预期不符，测试失败
+            expect(item[0]).toEqual(res[index][0]);
+            // 置信度对比，允许误差0.02
+            expect(Math.abs(item[1] - res[index][1])).toBeLessThanOrEqual(gap);
+            // 检测顶点坐标对比，允许误差0.02
+            // left
+            expect(Math.abs(item[2] - res[index][2])).toBeLessThanOrEqual(gap);
+            // top
+            expect(Math.abs(item[3] - res[index][3])).toBeLessThanOrEqual(gap);
+            // right
+            expect(Math.abs(item[4] - res[index][4])).toBeLessThanOrEqual(gap);
+            // bottom
+            expect(Math.abs(item[5] - res[index][5])).toBeLessThanOrEqual(gap);
+        });
+    });
+});
diff --git a/packages/paddlejs-core/src/mediaProcessor.ts b/packages/paddlejs-core/src/mediaProcessor.ts
@@ -147,7 +147,7 @@ export default class MediaProcessor {
     /**
      * 缩放成目标尺寸, keepRatio 为 true 则保持比例拉伸并居中，为 false 则变形拉伸为目标尺寸
      */
-    fitToTargetSize(image, imageDataInfo, opt) {
+    fitToTargetSize(image, imageDataInfo, opt?) {
         const {
             keepRatio = true,
             inGPU = false,
@@ -244,4 +244,4 @@ export default class MediaProcessor {
         const scale = [sw / dw, sh / dh];
         return scale;
     }
-}
+}
diff --git a/packages/paddlejs-models/detect/README.md b/packages/paddlejs-models/detect/README.md
@@ -0,0 +1,35 @@
+[中文版](./README_cn.md)
+
+# detect
+
+detect model is used to detect the position of label frame in the image.
+
+# Usage
+
+```js
+import * as det from '@paddlejs-models/detect';
+
+// Load model
+await det.load();
+
+// Get label index, confidence and coordinates
+const res = await det.detect(img);
+
+res.forEach(item => {
+    // Get label index
+    console.log(item[0]);
+    // Get label confidence
+    console.log(item[1]);
+    // Get label left coordinates
+    console.log(item[2]);
+    // Get label top coordinates
+    console.log(item[3]);
+    // Get label right coordinates
+    console.log(item[4]);
+    // Get label bottom coordinates
+    console.log(item[5]);
+});
+```
+
+# effect
+![img.png](https://user-images.githubusercontent.com/43414102/153805288-80f289bf-ca92-4788-b1dd-44854681a03f.png)
diff --git a/packages/paddlejs-models/detect/README_cn.md b/packages/paddlejs-models/detect/README_cn.md
@@ -0,0 +1,35 @@
+[English](./README.md)
+
+# detect
+
+detect模型用于检测图像中label框选位置。
+
+# 使用
+
+```js
+import * as det from '@paddlejs-models/detect';
+
+// 模型加载
+await det.load();
+
+// 获取label对应索引、置信度、检测框选坐标
+const res = await det.detect(img);
+
+res.forEach(item => {
+    // 获取label对应索引
+    console.log(item[0]);
+    // 获取label置信度
+    console.log(item[1]);
+    // 获取检测框选left顶点
+    console.log(item[2]);
+    // 获取检测框选top顶点
+    console.log(item[3]);
+    // 获取检测框选right顶点
+    console.log(item[4]);
+    // 获取检测框选bottom顶点
+    console.log(item[5]);
+});
+```
+
+# 效果
+![img.png](https://user-images.githubusercontent.com/43414102/153805288-80f289bf-ca92-4788-b1dd-44854681a03f.png)
diff --git a/packages/paddlejs-models/detect/demo/index.html b/packages/paddlejs-models/detect/demo/index.html
@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>paddle detection demo</title>
+    <meta name="viewport" content="width=device-width,minimum-scale=1.0,maximum-scale=1.0,user-scalable=no">
+    <style>
+        body {
+            margin: 0 auto;
+            width: 100%;
+            height: 100%;
+        }
+        .wrapper {
+            position: relative;
+            width: 100%;
+            height: 100%;
+            margin: 0 auto;
+        }
+        #isLoading {
+            position: fixed;
+            top: 0;
+            left: 0;
+            right: 0;
+            bottom: 0;
+            width: 100vw;
+            height: 100vh;
+            background-color: rgba(0, 0, 0, .5);
+        }
+        #isLoading .loading-text {
+            color: #fff;
+            font-size: 24px;
+            text-align: center;
+            line-height: 100vh;
+        }
+    </style>
+</head>
+<body>
+<div class="wrapper">
+    <img id="image" src="https://m.baidu.com/se/static/img/iphone/logo.png">
+    <div id="tool">
+        <input type="file" id="uploadImg">
+    </div>
+    <canvas id="canvas"></canvas>
+</div>
+
+<div id="isLoading">
+    <p class="loading-text center">loading……</p>
+</div>
+
+</body>
+</html>
diff --git a/packages/paddlejs-models/detect/demo/index.ts b/packages/paddlejs-models/detect/demo/index.ts
@@ -0,0 +1,51 @@
+import * as det from '../src/index';
+import label from './label.json';
+
+const loading = document.getElementById('isLoading');
+const inputElement = document.getElementById('uploadImg');
+const imgElement = document.getElementById('image') as HTMLImageElement;
+const canvasOutput = document.getElementById('canvas') as HTMLCanvasElement;
+
+let isPreheat = true;
+
+load();
+
+async function load() {
+    await det.init();
+    loading.style.display = 'none';
+    isPreheat = false;
+}
+
+inputElement.addEventListener('change', (e: Event) => {
+    imgElement.src = URL.createObjectURL((e.target as HTMLInputElement).files[0]);
+}, false);
+
+imgElement.onload = async () => {
+    if (isPreheat) {
+        return;
+    }
+    // 获取检测值
+    const res = await det.detect(imgElement);
+    const imgHeight = imgElement.height;
+    const imgWidth = imgElement.width;
+    canvasOutput.width = imgWidth;
+    canvasOutput.height = imgHeight;
+    const ctx = canvasOutput.getContext('2d');
+    ctx.drawImage(imgElement, 0, 0, canvasOutput.width, canvasOutput.height);
+    ctx.beginPath();
+    ctx.lineWidth = 1;
+    ctx.strokeStyle = 'red';
+    res.forEach(item => {
+        // 获取检测框选坐标
+        const left = Math.floor(item[2] * imgWidth);
+        const top = Math.floor(item[3] * imgHeight);
+        const right = Math.floor(item[4] * imgWidth);
+        const bottom = Math.floor(item[5] * imgHeight);
+        ctx.beginPath();
+        // 绘制检测框选矩形
+        ctx.rect(left, top, right - left, bottom - top);
+        // 绘制label
+        ctx.fillText(label[item[0]], left + 10, top + 10);
+        ctx.stroke();
+    });
+};
diff --git a/packages/paddlejs-models/detect/demo/label.json b/packages/paddlejs-models/detect/demo/label.json
@@ -0,0 +1,5 @@
+{
+    "0": "default",
+    "1": "screw",
+    "2": "nut"
+}