fix(webgl): fix prior_box and upload img to texture in WEBGL1.0 GLSL …

…ES 1.0
JingyuanZhang · Mar 22, 2022 · 2b5661a · 2b5661a
1 parent 2df6248
commit 2b5661a
Show file tree

Hide file tree

Showing 13 changed files with 112 additions and 86 deletions.
diff --git a/e2e/dist/assets/imgs/human.jpg b/e2e/dist/assets/imgs/human.jpg
diff --git a/e2e/dist/assets/imgs/seg.jpg b/e2e/dist/assets/imgs/seg.jpg
diff --git a/e2e/dist/assets/imgs/seg_398x224.png b/e2e/dist/assets/imgs/seg_398x224.png
diff --git a/e2e/dist/assets/imgs/seg_res.png b/e2e/dist/assets/imgs/seg_res.png
diff --git a/e2e/dist/index.html b/e2e/dist/index.html
@@ -30,8 +30,8 @@
         <img id="car" src="./assets/imgs/car.webp"/>
         <img id="banana" src="./assets/imgs/banana.jpeg"/>
         <img id="ok" src="./assets/imgs/ok.jpeg"/>
-        <img id="human" src="./assets/imgs/human.jpg"/>
-        <img id="seg" src="./assets/imgs/seg_398x224.png"/>
+        <img id="human" src="./assets/imgs/seg.jpg"/>
+        <img id="seg" src="./assets/imgs/seg_res.png"/>
         <img id="ocr" src="./assets/imgs/ocr.jpg"/>
         <img id="detect" src="./assets/imgs/detect.jpeg"/>
         <canvas id="back_canvas" ></canvas>

diff --git a/packages/paddlejs-backend-webgl/src/ops/atom/common_utils.ts b/packages/paddlejs-backend-webgl/src/ops/atom/common_utils.ts
diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/custom/imgFeed.ts b/packages/paddlejs-backend-webgl/src/ops/shader/custom/imgFeed.ts
@@ -5,12 +5,12 @@
 function mainFunc() {
     return `
     uniform vec2 u_scale;
-    uniform float u_keep_ratio;
+    uniform int u_keep_ratio;
 
     void main(void) {
         vec2 outCoord = vCoord.xy;
         // 支持模型不按比例拉伸
-        if (float(u_keep_ratio) == 0.0) {
+        if (u_keep_ratio > 0) {
             vec4 origin = TEXTURE2D(texture_origin, outCoord);
             setPackedOutput(origin);
             return;
@@ -26,7 +26,7 @@ function mainFunc() {
             setPackedOutput(origin);
         }
         else {
-            setPackedOutput(vec4(1.0, 1.0, 1.0, 1.0));
+            setPackedOutput(vec4(1.0));
         }
     }
     `;

diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/prior_box.ts b/packages/paddlejs-backend-webgl/src/ops/shader/prior_box.ts
@@ -6,7 +6,7 @@
 
 /* eslint-disable max-lines-per-function */
 
-import { genFpDataCode, genFpFloatArr } from '../../utils/dataProcess';
+import { genFpDataCode, genGLSLArr, ArrTypeEnum, getValueFromArrByIndex } from '../../utils/dataProcess';
 
 function mainFunc(
     { origin, image, out },
@@ -56,13 +56,18 @@ function mainFunc(
     // 这里没有使用 genFpDataCode 的原因是 aspect_ratios 长度很有可能超过 4
     // 但是 vec 最大是 vec4，所以用 float arr 更保险
     const floatArraydataCode = `
-        ${genFpFloatArr(min_sizes, 'min_sizes')}
-        ${genFpFloatArr(max_sizes, 'max_sizes')}
-        ${genFpFloatArr(aspect_ratios, 'aspect_ratios')}
+        ${genGLSLArr(min_sizes, 'min_sizes', ArrTypeEnum.FLOAT_TYPE)}
+        ${genGLSLArr(max_sizes, 'max_sizes', ArrTypeEnum.FLOAT_TYPE)}
+        ${genGLSLArr(aspect_ratios, 'aspect_ratios', ArrTypeEnum.FLOAT_TYPE)}
     `;
 
-    const clipCode = clip ? 'res = min(max(res, 0.), 1.);' : '';
+    const getValueFromArrByIndexGLSL = `
+        ${getValueFromArrByIndex(min_sizes, 'min_sizes', ArrTypeEnum.FLOAT_TYPE)}
+        ${getValueFromArrByIndex(max_sizes, 'max_sizes', ArrTypeEnum.FLOAT_TYPE)}
+        ${getValueFromArrByIndex(aspect_ratios, 'aspect_ratios', ArrTypeEnum.FLOAT_TYPE)}
+    `;
 
+    const clipCode = clip ? 'res = min(max(res, 0.), 1.);' : '';
 
 
     const prefix = `
@@ -81,6 +86,8 @@ function mainFunc(
         }
     }
 
+    ${getValueFromArrByIndexGLSL}
+
     // start函数
     void main(void) {
         ivec4 oPos = getOutputTensorPos();
@@ -119,26 +126,26 @@ function mainFunc(
 
             // 求idx 对应的 h w p m
             int h = int(idx / (num_priors * feature_width));
-            int w = (idx / num_priors) % feature_width;
-            int p = idx % num_priors;
+            int w = calMod(idx / num_priors, feature_width);
+            int p = calMod(idx, num_priors);
             int m = ${max_sizes.length > 0} ? int(p / (as_num + 1)) : int(p / as_num);
             float cx = (float(w) + offset) * step_width;
             float cy = (float(h) + offset) * step_height;
-            float min_size = float(min_sizes[m]);
+            float min_size = getValueFromArrByIndex_min_sizes(min_sizes, m);
             float bw = 0.0;
             float bh = 0.0;
 
             ${max_sizes.length > 0
         ? `
-            int s = p % (as_num + 1);
+            int s = calMod(p, as_num + 1);
             if (${!min_max_aspect_ratios_order}) {
                 if (s < as_num) {
-                    float ar = aspect_ratios[s];
+                    float ar = getValueFromArrByIndex_aspect_ratios(aspect_ratios, s);
                     bw = min_size * ar / 2.0;
                     bh = min_size / ar / 2.0;
                 }
                 else {
-                    float max_size = float(max_sizes[m]);
+                    float max_size = getValueFromArrByIndex_max_sizes(max_sizes, m);
                     bw = sqrt(min_size * max_size) / 2.0;
                     bh = bw;
                 }
@@ -149,19 +156,19 @@ function mainFunc(
                     bw = bh;
                 }
                 else if (s == 1) {
-                    float max_size = float(max_sizes[m]);
+                    float max_size = getValueFromArrByIndex_max_sizes(max_sizes, m);
                     bw = sqrt(min_size * max_size) / 2.0;
                     bh = bw;
                 }
                 else {
-                    float ar = aspect_ratios[s - 1];
+                    float ar = getValueFromArrByIndex_aspect_ratios(aspect_ratios, s - 1);
                     bw = min_size * sqrt(ar) / 2.0;
                     bh = min_size / sqrt(ar) / 2.0;
                 }
             }`
         : `
-            int s = p % as_num;
-            float ar = aspect_ratios[s];
+            int s = calMod(p, as_num);
+            float ar = getValueFromArrByIndex_aspect_ratios(aspect_ratios, s);
             bw = min_size * ar / 2.0;
             bh = min_size / ar / 2.0;
         `}

diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/slice.ts b/packages/paddlejs-backend-webgl/src/ops/shader/slice.ts
@@ -4,8 +4,7 @@
  * @example x = [[1,2,3,4],[5,6,7,8]] axes=[1] starts=[2] ends=[3] => out [3,7]
  */
 
-import { env } from '@paddlejs/paddlejs-core';
-import { initializeGLSLArr, ArrTypeEnum } from '../atom/common_utils';
+import { genGLSLArr, ArrTypeEnum, getValueFromArrByIndex } from '../../utils/dataProcess';
 
 
 function mainFunc(
@@ -75,27 +74,13 @@ function mainFunc(
         }
     }
 
-    const glslIndexArr = initializeGLSLArr(res_pos, ArrTypeEnum.INT_TYPE);
-
-    const ifConditions = res_pos.reduce((acc, _, idx) => {
-        const ifCondition = idx === 0
-            ? `
-                int index = 0;
-                if (sumVal == ${idx}) {
-                    index = arr[${idx}];
-                }`
-            : `
-                else if (sumVal == ${idx}) {
-                    index = arr[${idx}];
-                }
-            `;
-        return acc + ifCondition;
-    }, '');
+    const glslIndexArr = genGLSLArr(res_pos, 'arr', ArrTypeEnum.INT_TYPE);
+
+    const getValueFromArrByIndexGLSL = getValueFromArrByIndex(res_pos, 'arr', ArrTypeEnum.INT_TYPE);
 
-    const getValueFromArrIndex = env.get('webglVersion') === 2
-        ? 'int index = arr[sumVal];'
-        : ifConditions;
     return `
+    ${getValueFromArrByIndexGLSL}
+
     void main(void) {
         ivec4 oPos = getOutputTensorPos();
         ${glslIndexArr}
@@ -105,8 +90,8 @@ function mainFunc(
             + oPos.b * ${out.width_shape}
             + oPos.g * ${out.height_shape} * ${out.width_shape}
             + oPos.r * ${out.channel} * ${out.width_shape} * ${out.height_shape};
-        
-        ${getValueFromArrIndex}
+
+        int index = getValueFromArrByIndex_arr(arr, sumVal);
 
         float res = 0.0;
         ivec4 co = getTensorPosFromArrayIndex_origin(index);

diff --git a/packages/paddlejs-backend-webgl/src/utils/dataProcess.ts b/packages/paddlejs-backend-webgl/src/utils/dataProcess.ts
@@ -1,6 +1,9 @@
 /**
  * @file data process
  */
+
+import { env } from '@paddlejs/paddlejs-core';
+
 function nhwc2nchw(data: number[] | Float32Array, shape: number[]) {
     const N = shape[0];
     const H = shape[1];
@@ -38,7 +41,7 @@ function getSizeFromShape(shape: number[]): number {
     return shape.reduce((acc, cur) => acc * cur, 1);
 }
 
-function genFpFloatArr(arr, key) {
+function genGLSLFloatArr(arr, key) {
     if (arr.length === 0) {
         return '';
     }
@@ -95,12 +98,71 @@ function getSmallestDivisor(number, base) {
     return divisor;
 }
 
+enum ArrTypeEnum {
+    INT_TYPE = 'int',
+    FLOAT_TYPE = 'float'
+}
+
+// GLSL ES 3.00 支持 arr =>  int arr = int[](x, x, x,... x);
+// GLSL ES 1.0 (1.0) 不支持  array constructor
+// '[]' : array constructor supported in GLSL ES 3.00 and above only
+function genGLSLArr(arr: Array<Number>, key: string, type: ArrTypeEnum) {
+    if (arr.length === 0) {
+        return '';
+    }
+
+    if (env.get('webglVersion') === 2) {
+        return arr.reduce((acc, cur, index) => {
+            const tmp = index < arr.length - 1 ? `${type}(${cur}), ` : `${type}(${cur}));`;
+            return acc + tmp;
+        }, `${type} ${key}[] = ${type}[](`);
+    }
+
+    const arr_value = arr.reduce((acc, cur, index) => {
+        return acc + `
+            ${key}[${index}] = ${type}(${cur});`;
+    }, '');
+
+    return `
+        ${type} ${key}[${arr.length}];
+        ${arr_value}
+    `;
+}
+
+function getValueFromArrByIndex(arr: Array<number>, arrKey: string, type: ArrTypeEnum) {
+    const ifConditions = arr.reduce((acc, _, idx) => {
+        const ifCondition = idx === 0
+            ? `
+            ${type} res = ${type}(0);
+            if (index == ${idx}) {
+                res = arr[${idx}];
+            }`
+            : `
+            else if (index == ${idx}) {
+                res = arr[${idx}];
+            }`;
+        return acc + ifCondition;
+    }, '');
+
+    return `
+    ${type} getValueFromArrByIndex_${arrKey}(${type}[${arr.length}] arr, int index) {
+        ${env.get('webglVersion') === 2
+        ? `${type} res = arr[index];`
+        : ifConditions}
+        return res;
+    }
+    `;
+}
+
 export {
     nhwc2nchw,
     getSizeFromShape,
     reduceShape,
     genFpDataCode,
-    genFpFloatArr,
+    genGLSLFloatArr,
     genIntDataCode,
-    getSmallestDivisor
+    getSmallestDivisor,
+    genGLSLArr,
+    ArrTypeEnum,
+    getValueFromArrByIndex
 };
diff --git a/packages/paddlejs-backend-webgl/src/webgl/WebGLTexture.ts b/packages/paddlejs-backend-webgl/src/webgl/WebGLTexture.ts
@@ -181,6 +181,15 @@ export class GLTexture {
             // case1: 输入为0~255之间的像素数据，类型为Uint8Array 或 Uint8ClampedArray
             // case2: 小程序环境，输入数据可能是HTMLImageElement、HTMLVideoElement、HTMLCanvasElement、小程序中图像的临时path string。
             texeltype = gl.UNSIGNED_BYTE;
+            gl.texImage2D(
+                gl.TEXTURE_2D,
+                0,
+                gl.RGBA,
+                gl.RGBA,
+                gl.UNSIGNED_BYTE,
+                data
+            );
+            return;
         }
         else if (env.get('webglVersion') === 2) {
             // 输入数据类型是Float32Array

diff --git a/packages/paddlejs-core/src/transform/feedProcess.ts b/packages/paddlejs-core/src/transform/feedProcess.ts
@@ -97,7 +97,7 @@ export default class WebglFeedProcess extends Transformer {
                     value: [0, 0]
                 },
                 u_keep_ratio: {
-                    type: UniformType.uniform1f,
+                    type: UniformType.uniform1i,
                     value: 0
                 }
             },

diff --git a/packages/paddlejs-models/detect/src/index.ts b/packages/paddlejs-models/detect/src/index.ts
@@ -14,9 +14,10 @@ export async function init() {
         mean: [0.5, 0.5, 0.5],
         std: [0.5, 0.5, 0.5],
         bgr: true,
-        keepRatio: false,
+        keepRatio: true,
         webglFeedProcess: true
     });
+
     await detectRunner.init();
 }