diff --git a/packages/paddlejs-backend-webgl/package.json b/packages/paddlejs-backend-webgl/package.json index 85f7a1cb..50e715be 100644 --- a/packages/paddlejs-backend-webgl/package.json +++ b/packages/paddlejs-backend-webgl/package.json @@ -1,6 +1,6 @@ { "name": "@paddlejs/paddlejs-backend-webgl", - "version": "1.2.9", + "version": "1.2.10-beta.0", "description": "", "main": "lib/index", "scripts": { diff --git a/packages/paddlejs-backend-webgl/src/backend.ts b/packages/paddlejs-backend-webgl/src/backend.ts index f96fbbcd..df0383b7 100644 --- a/packages/paddlejs-backend-webgl/src/backend.ts +++ b/packages/paddlejs-backend-webgl/src/backend.ts @@ -409,7 +409,13 @@ export default class WebGLBackend extends PaddlejsBackend { return loc; } - dispose() { - + dispose({ inputTensors }) { + for (const tensor of inputTensors) { + const tensorName = tensor.tensorId; + this.texturesMap[tensorName] && this.gl.deleteTexture(this.texturesMap[tensorName]); + this.cacheTextures[tensorName] && this.gl.deleteTexture(this.cacheTextures[tensorName]); + this.cacheTextures[tensorName] = null; + this.texturesMap[tensorName] = null; + } } } diff --git a/packages/paddlejs-backend-webgl/src/ops/atom/common_func.ts b/packages/paddlejs-backend-webgl/src/ops/atom/common_func.ts index 82dd5d6c..b163d3b5 100644 --- a/packages/paddlejs-backend-webgl/src/ops/atom/common_func.ts +++ b/packages/paddlejs-backend-webgl/src/ops/atom/common_func.ts @@ -75,6 +75,11 @@ float exp_func(float x, float y, float z) { return result; }`; +const abs_func = ` +float abs_func(float x, float y, float z) { + return abs(x); +}`; + export { prelu, relu6, @@ -87,6 +92,7 @@ export { sqrt, pow_func, tanh_func, + abs_func, transferFromNHWCtoNCHW }; diff --git a/packages/paddlejs-backend-webgl/src/ops/atom/fuse_ops.ts b/packages/paddlejs-backend-webgl/src/ops/atom/fuse_ops.ts index 4009b0b1..8e9bc62f 100644 --- a/packages/paddlejs-backend-webgl/src/ops/atom/fuse_ops.ts +++ b/packages/paddlejs-backend-webgl/src/ops/atom/fuse_ops.ts @@ -93,6 +93,10 @@ export default function genFuseOpCode(params: OpParams) { act_name = 'exp_func'; break; + case 'abs': + act_name = 'abs_func'; + break; + default: break; } @@ -127,7 +131,7 @@ export default function genFuseOpCode(params: OpParams) { return ` ${activation_func} - + float fuse_op(float x) { float res = x; ${calculation_str} diff --git a/packages/paddlejs-backend-webgl/src/ops/index.ts b/packages/paddlejs-backend-webgl/src/ops/index.ts index dcebd562..edfadb90 100644 --- a/packages/paddlejs-backend-webgl/src/ops/index.ts +++ b/packages/paddlejs-backend-webgl/src/ops/index.ts @@ -26,6 +26,8 @@ import batchnorm from './shader/batchnorm'; import reshape2 from './shader/reshape2'; import bilinear_interp from './shader/bilinear_interp'; import bilinear_interp_v2 from './shader/bilinear_interp_v2'; +import linear_interp from './shader/linear_interp'; +import linear_interp_v2 from './shader/linear_interp_v2'; import transpose2 from './shader/transpose2'; import softmax from './shader/softmax'; import dynamic from './shader/dynamic'; @@ -36,11 +38,12 @@ import arg_min from './shader/arg_min'; import unsqueeze2 from './shader/unsqueeze2'; import flatten_contiguous_range from './shader/flatten_contiguous_range'; import greater_than from './shader/greater_than'; -import reduce_sum from './shader/reduce_sum'; import where from './shader/where'; import connect from './shader/connect'; import squeeze2 from './shader/squeeze2'; import pad3d from './shader/pad3d'; +import reduce_sum from './shader/reduce_sum'; +import reduce_max from './shader/reduce_max'; import reduce_mean from './shader/reduce_mean'; import shuffle_channel from './shader/shuffle_channel'; import hard_swish from './shader/hard_swish'; @@ -97,6 +100,7 @@ const ops = { reshape: reshape2, reshape2, bilinear_interp, + linear_interp, transpose2, unpacked_2_packed, packed_2_unpacked, @@ -105,9 +109,10 @@ const ops = { flatten2: reshape2, greater_than, reduce_sum, + reduce_mean, + reduce_max, where, connect, - reduce_mean, hard_swish, nearest_interp, nearest_interp_v2, @@ -130,9 +135,11 @@ const ops = { sqrt: dynamic('sqrt'), tanh: dynamic('tanh'), exp: dynamic('exp'), + abs: dynamic('abs'), squeeze2, pad3d, bilinear_interp_v2, + linear_interp_v2, shuffle_channel, pack_out, nhwc_2_nchw, diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/batchnorm.ts b/packages/paddlejs-backend-webgl/src/ops/shader/batchnorm.ts index 996e1f53..dbaf9b36 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/batchnorm.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/batchnorm.ts @@ -3,9 +3,13 @@ */ function mainFunc( - { bias, scale, mean, variance }, + { bias, scale, mean, variance, origin }, { epsilon } ) { + // calc the real pos of channel + const len = origin.length_unformatted_shape; + const realCPos = 4 - len + 1; + const posStr = `oPos[${realCPos}]`; return ` // start函数 void main(void) { @@ -14,11 +18,14 @@ function mainFunc( float o = getValueFromTensorPos_origin(oPos.r, oPos.g, oPos.b, oPos.a); // 归一化数据 - vec4 scale = getPixelsFromTexturePos_scale(vec2( float(oPos.g) / float(${scale.width_texture}) + 0.00001, 0.0)); - vec4 bias = getPixelsFromTexturePos_bias(vec2( float(oPos.g) / float(${bias.width_texture}) + 0.00001, 0.0)); - vec4 mean = getPixelsFromTexturePos_mean(vec2((float(oPos.g)) / float(${mean.width_texture}) + 0.00001, 0.0)); + vec4 scale = getPixelsFromTexturePos_scale( + vec2(float(${posStr}) / float(${scale.width_texture}) + 0.00001, 0.0)); + vec4 bias = getPixelsFromTexturePos_bias( + vec2(float(${posStr}) / float(${bias.width_texture}) + 0.00001, 0.0)); + vec4 mean = getPixelsFromTexturePos_mean( + vec2((float(${posStr})) / float(${mean.width_texture}) + 0.00001, 0.0)); vec4 variance = getPixelsFromTexturePos_variance( - vec2((float(oPos.g)) / float(${variance.width_texture}) + 0.00001, + vec2((float(${posStr})) / float(${variance.width_texture}) + 0.00001, 0.0) ); diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/dynamic.ts b/packages/paddlejs-backend-webgl/src/ops/shader/dynamic.ts index 45bc8886..da937cd7 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/dynamic.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/dynamic.ts @@ -13,7 +13,8 @@ const commonFuncBehaviors = { pow: ['transToPow'], exp: ['transToExp'], sqrt: ['transToSqrt'], - tanh: ['transToTanh'] + tanh: ['transToTanh'], + abs: ['transToAbs'] }; function mainFunc( diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/linear_interp.ts b/packages/paddlejs-backend-webgl/src/ops/shader/linear_interp.ts new file mode 100644 index 00000000..b2c57522 --- /dev/null +++ b/packages/paddlejs-backend-webgl/src/ops/shader/linear_interp.ts @@ -0,0 +1,58 @@ +/** + * @file bilinear_interp + */ + +function mainFunc( + { out, origin }, + { align_mode = 1, align_corners = true } +) { + const outW = out.width_shape; + const inW = origin.width_shape; + const scale = align_corners ? (outW - 1) / (inW - 1) : (outW / inW); + + return ` + // start函数 + + vec4 getData(float n, float scale, bool align_flag, int in_len) { + float m = align_flag ? ((n + 0.5) / scale - 0.5) : (n / scale); + // int a1 = int(floor(m + 0.5)); + int a1 = int(m); + a1 = a1 > 0 ? a1 : 0; + int a2 = a1 < (in_len - 1) ? (a1 + 1) : in_len; + + float idx_src = (n + 0.5) / scale - 0.5; + idx_src = idx_src > 0.0 ? idx_src : 0.0; + float b1 = align_flag ? (idx_src - float(a1)) : (n / scale - float(a1)); + float b2 = 1.0 - b1; + return vec4(float(a1), float(a2), b1, b2); + } + + void main(void) { + // 输出数据 + ivec4 oPos = getOutputTensorPos(); + + bool align_flag = ${align_mode} == 0 && !${align_corners}; + + float scale = float(${scale}); + + vec4 v = getData(float(oPos.a), scale, align_flag, ${inW}); + + float v1 = v.r; + float v2 = v.g; + float v3 = v.b; + float v4 = v.a; + + float value1 = getValueFromTensorPos_origin(oPos.r, oPos.g, oPos.b, int(v1)); + float value2 = getValueFromTensorPos_origin(oPos.r, oPos.g, oPos.b, int(v2)); + float value = v4 * value1 + v3 * value2; + // setOutput(float(int(v1))); + setOutput(float(value)); + } + `; +} +export default { + mainFunc, + textureFuncConf: { + origin: ['getValueFromTensorPos'] + } +}; diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/linear_interp_v2.ts b/packages/paddlejs-backend-webgl/src/ops/shader/linear_interp_v2.ts new file mode 100644 index 00000000..7a2f5499 --- /dev/null +++ b/packages/paddlejs-backend-webgl/src/ops/shader/linear_interp_v2.ts @@ -0,0 +1,8 @@ +/** + * @file bilinear_interp_v2 + * @implements 差值实现方式(BilinearInterpolation)与 bilinear_interp 一致,唯一的区别是 outshape 计算 + */ + +import linear_interp from './linear_interp'; + +export default linear_interp; \ No newline at end of file diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/pool2d_max.ts b/packages/paddlejs-backend-webgl/src/ops/shader/pool2d_max.ts index ce537b9c..beed01ab 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/pool2d_max.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/pool2d_max.ts @@ -22,7 +22,7 @@ function mainFunc( index = ${originShape[2] * originShape[3]} * out_pos[1] + ${originShape[3]} * oy + ox; } `; - outputCode = 'setOutput(float(index));'; + outputCode = 'setOutput(float(res));'; } return ` // start函数 diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/reduce_max.ts b/packages/paddlejs-backend-webgl/src/ops/shader/reduce_max.ts new file mode 100644 index 00000000..8f870b55 --- /dev/null +++ b/packages/paddlejs-backend-webgl/src/ops/shader/reduce_max.ts @@ -0,0 +1,32 @@ + +/** + * @file concat + */ + +function mainFunc( + {}, + { inputs_dim, dim } +) { + return ` + // start函数 + void main(void) { + ivec4 oPos = getOutputTensorPos(); + // 输出坐标转换为输入坐标 + float o = 0.0; + for (int i = 0; i < ${inputs_dim}; i++) { + oPos[${dim}] = i; + o = max(o, getValueFromTensorPos_origin(oPos.r, oPos.g, oPos.b, oPos.a)); + } + setOutput(float(o)); + } + `; +} +export default { + mainFunc, + textureFuncConf: { + origin: ['getValueFromTensorPos'] + }, + behaviors: [ + 'normalizeDim' + ] +}; diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/reshape2.ts b/packages/paddlejs-backend-webgl/src/ops/shader/reshape2.ts index 8c5ac093..3e853b74 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/reshape2.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/reshape2.ts @@ -3,23 +3,35 @@ */ function mainFunc( - { out }, + { origin, out }, {} ) { return ` // start函数 void main(void) { - vec2 outCoord = vCoord.xy * (_2d_shape_texture_out); - int index = int(outCoord.x) + int(outCoord.y) * int(${out.width_texture}); - ivec4 originPos = getTensorPosFromArrayIndex_origin(index); - float res = getValueFromTensorPos_origin(originPos[0], originPos[1], originPos[2], originPos[3]); - setOutput(res); + // 输出数据 + ivec4 oPos = getOutputTensorPos(); + // 输出坐标转换为输入坐标 + int sumVal = oPos.a + + oPos.b * ${out.width_shape} + + oPos.g * ${out.height_shape} * ${out.width_shape} + + oPos.r * ${out.channel} * ${out.width_shape} * ${out.height_shape}; + ivec4 new_oPos = transferFromNHWCtoNCHW( + sumVal, + ${origin.channel}, + ${origin.width_shape}, + ${origin.height_shape}, + ${origin.total_shape} + ); + float o = getValueFromTensorPos_origin(new_oPos.r, new_oPos.g, new_oPos.b, new_oPos.a); + setOutput(float(o)); } `; } export default { mainFunc, textureFuncConf: { - origin: ['getTensorPosFromArrayIndex', 'getValueFromTensorPos'] - } + origin: ['getValueFromTensorPos'] + }, + commonFuncConf: ['transferFromNHWCtoNCHW'] }; diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/slice.ts b/packages/paddlejs-backend-webgl/src/ops/shader/slice.ts index 407b5566..0a991b87 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/slice.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/slice.ts @@ -4,97 +4,46 @@ * @example x = [[1,2,3,4],[5,6,7,8]] axes=[1] starts=[2] ends=[3] => out [3,7] */ -import { genGLSLArr, ArrTypeEnum, getValueFromArrByIndex } from '../../utils/dataProcess'; - - function mainFunc( { - out, origin + origin }, { - axes, starts, - ends, - decrease_axis + axes } ) { - - if ( - axes.length > 1 - || starts.length > 1 - || ends.length > 1 - || (decrease_axis && decrease_axis.length === 0) - ) { - throw Error('[slice op feature]: current support one dim, support decrease_axis'); - } const { + length_unformatted_shape, width_shape, height_shape, - channel, - total_shape, - length_unformatted_shape + channel } = origin; - const batch = total_shape / (width_shape * height_shape * channel); - const tensor_shape = [batch, channel, height_shape, width_shape]; - let axis = axes[0]; - - if (axis < 0) { - axis = axis + length_unformatted_shape + 1; - } - axis = 4 - length_unformatted_shape + axis; - - if (axis !== 4) { - throw Error('[slice op feature]: unsupport axis value'); + const axisList = Array.isArray(axes) ? axes : [axes]; + if (axisList.length !== 1) { + throw Error('[slice op feature]: axes only support one dim'); } + let axis = axisList[0]; + axis = axis === -1 ? 3 : (4 - length_unformatted_shape + axis); + // const curDim = tensor_shape[axis]; + const originPos = ['(oPos[0])', '(oPos[1])', '(oPos[2])', '(oPos[3])']; + originPos[axis] = `(oPos[${axis}]+ start)`; const start = starts[0]; - const end = ends[0]; - - const [ - batch_num, - channel_num, - height_num, - width_num - ] = tensor_shape; - - // 计算 output tensor value 对应的 origin index - const res_pos = []; - for (let index = start; index < end; index++) { - for (let batch = 0; batch < batch_num; batch++) { - for (let channel = 0; channel < channel_num; channel++) { - for (let height = 0; height < height_num; height++) { - res_pos.push( - batch * channel_num * height_num * width_num - + channel * height_num * width_num - + height * width_num + index - ); - } - } - } - } - - const glslIndexArr = genGLSLArr(res_pos, 'arr', ArrTypeEnum.INT_TYPE); - - const getValueFromArrByIndexGLSL = getValueFromArrByIndex(res_pos, 'arr', ArrTypeEnum.INT_TYPE); return ` - ${getValueFromArrByIndexGLSL} - void main(void) { + int start = ${start}; ivec4 oPos = getOutputTensorPos(); - ${glslIndexArr} - // 输出坐标转换为输入坐标 - int sumVal = oPos.a - + oPos.b * ${out.width_shape} - + oPos.g * ${out.height_shape} * ${out.width_shape} - + oPos.r * ${out.channel} * ${out.width_shape} * ${out.height_shape}; - - int index = getValueFromArrByIndex_arr(arr, sumVal); + int sumVal = ${originPos[3]} + + ${originPos[2]} * ${width_shape} + + ${originPos[1]} * ${height_shape * width_shape} + + ${originPos[0]} * ${channel * width_shape * height_shape}; float res = 0.0; - ivec4 co = getTensorPosFromArrayIndex_origin(index); + ivec4 co = getTensorPosFromArrayIndex_origin(sumVal); res = getValueFromTensorPos_origin(co.r, co.g, co.b, co.a); setOutput(float(res)); } @@ -103,6 +52,6 @@ function mainFunc( export default { mainFunc, textureFuncConf: { - origin: ['getValueFromTensorPos', 'getTensorPosFromArrayIndex'] + origin: ['getTensorPosFromArrayIndex', 'getValueFromTensorPos'] } }; diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/softmax.ts b/packages/paddlejs-backend-webgl/src/ops/shader/softmax.ts index e9d64ed3..590b8f59 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/softmax.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/softmax.ts @@ -6,10 +6,10 @@ function mainFunc( { origin }, { axis } ) { - let axisVal = axis; - if (!axis || axis < 0) { - axisVal = (axis || -1) + 4; - } + const length_unformatted_shape = origin.length_unformatted_shape; + const axisVal = !axis || axis < 0 + ? ((axis || -1) + 4) + : axis + length_unformatted_shape; return ` // start函数 void main(void) { diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/squeeze2.ts b/packages/paddlejs-backend-webgl/src/ops/shader/squeeze2.ts index 7fbbf0bd..0f891594 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/squeeze2.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/squeeze2.ts @@ -3,19 +3,27 @@ */ function mainFunc( - {}, + { origin, out }, { axes } ) { + const length_unformatted_shape = origin.length_unformatted_shape; + const out_unformatted_shape = out.length_unformatted_shape; const axesArr = Array.isArray(axes) ? axes : [axes]; + const diffNum = 4 - length_unformatted_shape; + const newAxes = axesArr.map(item => item === -1 ? 3 : (item + diffNum)); + // 获取output的维度 - const posArr = [0, 1, 2, 3].filter(item => item >= axesArr.length); - const newPosArr = [0, 1, 2, 3].map(item => { - if (axesArr.indexOf(item) > -1) { - return 0; + // const posArr = [0, 1, 2, 3].filter(item => item >= axesArr.length); + const posArr = ['oPos.r', 'oPos.g', 'oPos.b', 'oPos.a'].slice(-1 * out_unformatted_shape); + const dest = [0, 1, 2, 3]; + Array.from(newAxes, axis => dest.splice(axis, 1, -1)); + const newPosArr = dest.reverse().map(item => { + if (item === -1) { + return '0'; } - const nowIndex = posArr.splice(0, 1); - return `oPos[${nowIndex}]`; - }); + return posArr.pop() || '0'; + }).reverse(); + const posStr = newPosArr.join(','); return ` diff --git a/packages/paddlejs-backend-webgl/src/ops/shader/unsqueeze2.ts b/packages/paddlejs-backend-webgl/src/ops/shader/unsqueeze2.ts index ef92af13..30a51bb4 100644 --- a/packages/paddlejs-backend-webgl/src/ops/shader/unsqueeze2.ts +++ b/packages/paddlejs-backend-webgl/src/ops/shader/unsqueeze2.ts @@ -4,7 +4,7 @@ function mainFunc( { - origin + out }, { axes @@ -12,19 +12,30 @@ function mainFunc( ) { const { length_unformatted_shape - } = origin; + } = out; const axes_arr = Array.isArray(axes) ? axes : [axes]; - const diffNum = 4 - length_unformatted_shape - axes_arr.length; - const newAxes = axes_arr.map(item => item + diffNum); + const newAxes = axes_arr.map(item => item === -1 ? 3 : item); // 获取 output shape 里原本对应 origin shape 的维度 - const posArr = [0, 1, 2, 3]; - const newPosArr = posArr.filter(item => newAxes.indexOf(item) === -1).map(item => `oPos[${item}]`); - const prefix = Array.from(new Array(newAxes.length), () => '0'); - newPosArr.splice(0, 0, ...prefix); - const posStr = newPosArr.join(','); + const posArr = [0, 1, 2, 3].slice(4 - length_unformatted_shape); + for (const axis of newAxes) { + posArr.splice(axis, 1); + } + + const prefixArr = Array.from(new Array(4 - posArr.length), () => -1); + const newPosArr = [...prefixArr, ...posArr]; + + const newPosStrArr = []; + + for (let i = 0, len = newPosArr.length; i < len; i++) { + const v = newPosArr[i]; + const str = v === -1 ? '0' : `oPos[${v}]`; + newPosStrArr.push(str); + } + + const posStr = newPosStrArr.join(','); return ` void main() { ivec4 oPos = getOutputTensorPos(); diff --git a/packages/paddlejs-core/package.json b/packages/paddlejs-core/package.json index 4ce99380..8791411a 100644 --- a/packages/paddlejs-core/package.json +++ b/packages/paddlejs-core/package.json @@ -1,6 +1,6 @@ { "name": "@paddlejs/paddlejs-core", - "version": "2.1.28", + "version": "2.1.29-beta.0", "description": "", "main": "lib/index", "scripts": { diff --git a/packages/paddlejs-core/src/commons/interface.ts b/packages/paddlejs-core/src/commons/interface.ts index 85e2620d..4c19c08c 100644 --- a/packages/paddlejs-core/src/commons/interface.ts +++ b/packages/paddlejs-core/src/commons/interface.ts @@ -50,6 +50,7 @@ export interface Model { chunkNum?: number; feedShape?: FeedShape | null; dataLayout?: string; + feed?: ModelVar; ops: ModelOp[]; vars: ModelVar[]; multiOutputs?: ModelOp[]; diff --git a/packages/paddlejs-core/src/graph.ts b/packages/paddlejs-core/src/graph.ts index c7ea553f..f357e376 100644 --- a/packages/paddlejs-core/src/graph.ts +++ b/packages/paddlejs-core/src/graph.ts @@ -101,27 +101,34 @@ export default class ModelGraph { } private arrangeMap() { - const executed: object = {}; const inIndex: number[] = []; const idtoindex: object = {}; + const indexMap: object = {}; for (let index = 0; index < this.weightMap.length; index++) { const item = this.weightMap[index]; - for (let index = 0; index < item.outputsName.length; index++) { - const outputName = item.outputsName[index]; - executed[outputName] = true; + for (let i = 0; i < item.outputsName.length; i++) { + const outputName = item.outputsName[i]; + if (!indexMap[outputName]) { + indexMap[outputName] = 1; + } + else { + indexMap[outputName]++; + } + idtoindex[item.id] = index; } + } + + for (let index = 0; index < this.weightMap.length; index++) { + const item = this.weightMap[index]; + inIndex[index] = 0; - idtoindex[item.id] = index; - if (item.inputsName.length > 1) { - item.inputsName.forEach(i => { - if (executed[i] === true) { - inIndex[index]++; - } - }); - } - else { - inIndex[index] = item.inputsName.length; + + for (let i = 0; i < item.inputsName.length; i++) { + const inputName = item.inputsName[i]; + if (indexMap[inputName]) { + inIndex[index]++; + } } } @@ -129,15 +136,37 @@ export default class ModelGraph { } private topoSort(ops: OpExecutor[], inIndex: number[], idtoindex: object) { - const inline: OpExecutor[] = []; - inline.push(ops[0]); - const ops_temp = ops.slice(0); - let prev: OpExecutor = null; - let iterator: OpExecutor = ops[0]; + const zeroIndexList = []; + const bpOp = ops.slice(0); + for (let i = 0; i < inIndex.length; i++) { + if (inIndex[i] === 0) { + zeroIndexList.push(ops[i]); + } + } + + let preOp = null; + for (const curOp of zeroIndexList) { + preOp = this.topoSortInner(ops, inIndex, idtoindex, curOp, bpOp, preOp); + } + } + + private topoSortInner( + ops: OpExecutor[], + inIndex: number[], + idtoindex: object, + curOp: OpExecutor, + ops_temp: OpExecutor[], + preOp?: OpExecutor + ) { + const inline = [curOp]; + let prev: OpExecutor = preOp; + let iterator: OpExecutor = curOp; + while (inline.length > 0) { - if (prev != null) { + if (prev) { ops[idtoindex[prev.id]].next = iterator.id; } + prev = iterator; iterator = inline.pop() || {} as OpExecutor; @@ -157,6 +186,9 @@ export default class ModelGraph { } } } + + ops[idtoindex[prev.id]].next = iterator.id; + return iterator; } /** diff --git a/packages/paddlejs-core/src/index.ts b/packages/paddlejs-core/src/index.ts index c14c3dee..2bf24d4b 100644 --- a/packages/paddlejs-core/src/index.ts +++ b/packages/paddlejs-core/src/index.ts @@ -1,10 +1,13 @@ import Runner from './runner'; import PaddlejsBackend from './backend'; -import { registerBackend, registerOp } from './globals'; +import { registerBackend, registerOp, GLOBALS } from './globals'; import Env from './env'; import * as interfaces from './commons/interface'; import Transformer from './transform/transformer'; import * as coreUtils from './commons/utils'; +import Tensor from './opFactory/tensor'; +import OpData from './opFactory/opDataBuilder'; +import { nhwc2nchw, nchw2nhwc } from './opFactory/utils'; export { Runner, @@ -14,5 +17,10 @@ export { interfaces, Transformer, Env as env, - coreUtils + coreUtils, + Tensor, + OpData, + GLOBALS, + nhwc2nchw, + nchw2nhwc }; diff --git a/packages/paddlejs-core/src/loader.ts b/packages/paddlejs-core/src/loader.ts index 0c5a5de1..729c0a2e 100644 --- a/packages/paddlejs-core/src/loader.ts +++ b/packages/paddlejs-core/src/loader.ts @@ -62,8 +62,8 @@ export default class ModelLoader { this.inNode = env.get('platform') === 'node'; } - async load() { - const modelInfo: Model = await this.fetchModel(); + async load(modelJSON) { + const modelInfo: Model = modelJSON || await this.fetchModel(); this.separateChunk = !!modelInfo.chunkNum && modelInfo.chunkNum > 0; this.chunkNum = this.separateChunk ? modelInfo.chunkNum : 0; diff --git a/packages/paddlejs-core/src/mediaProcessor.ts b/packages/paddlejs-core/src/mediaProcessor.ts index 76e860c3..63f48504 100644 --- a/packages/paddlejs-core/src/mediaProcessor.ts +++ b/packages/paddlejs-core/src/mediaProcessor.ts @@ -139,7 +139,7 @@ export default class MediaProcessor { } } - const nchwPixels: Float32Array = nhwc2nchw(result, [1, h, w, c]); + const nchwPixels: Float32Array = new Float32Array(nhwc2nchw(result, [1, h, w, c])); return nchwPixels; } @@ -213,7 +213,7 @@ export default class MediaProcessor { this.targetContext.fillStyle = imageDataInfo.gapFillWith; this.targetContext.fillRect(0, 0, this.targetCanvas.width, this.targetCanvas.height); - this.targetContext.drawImage(image, x, y, sw, sh); + this.targetContext.drawImage(image, 0, 0, sw, sh); } /** diff --git a/packages/paddlejs-core/src/opFactory/opBehaviors.ts b/packages/paddlejs-core/src/opFactory/opBehaviors.ts index fff6f873..614dbf25 100644 --- a/packages/paddlejs-core/src/opFactory/opBehaviors.ts +++ b/packages/paddlejs-core/src/opFactory/opBehaviors.ts @@ -151,6 +151,10 @@ const behaviors : Behaviors = { this.processedAttrs['active_function'] = 'tanh_func'; }, + transToAbs() { + this.processedAttrs['active_function'] = 'abs_func'; + }, + transToExp() { this.processedAttrs['active_function'] = 'exp'; }, @@ -217,7 +221,11 @@ const behaviors : Behaviors = { normalizeDim() { const originShape = this.tensorDataMap['origin'].shape; const shape = Utils.formatShape(originShape); - const axis = Utils.formatAxis(originShape, this.processedAttrs.axis); + const { axis: attrAxis, dim: attrDim } = this.processedAttrs; + const originAxis = attrAxis !== undefined + ? attrAxis + : (Array.isArray(attrDim) && attrDim.length ? attrDim[0] : attrDim); + const axis = Utils.formatAxis(originShape, originAxis); const dim_value: number[] = []; for (let index = 0; index < shape[axis]; index++) { dim_value[index] = index; diff --git a/packages/paddlejs-core/src/opFactory/utils.ts b/packages/paddlejs-core/src/opFactory/utils.ts index 5684f5ad..d9194660 100644 --- a/packages/paddlejs-core/src/opFactory/utils.ts +++ b/packages/paddlejs-core/src/opFactory/utils.ts @@ -87,6 +87,29 @@ export function packOpData(opData, packedName) { +/** + * 将nchw排布数据转为nhwc排布数据 + * @param {Array} data tensor data + * @param {Array} shape nchw + * @returns {Array} nhwc data + */ +export function nchw2nhwc(data: number[] | Float32Array, shape: number[]): number[] | Float32Array { + const [N, C, H, W] = shape; + const WXC = C * W; + const CXHXW = C * H * W; + const nhwcData: number[] | Float32Array = []; + for (let n = 0; n < N; n++) { + for (let c = 0; c < C; c++) { + for (let h = 0; h < H; h++) { + for (let w = 0; w < W; w++) { + nhwcData.push(data[n * CXHXW + h * WXC + w * C + c]); + } + } + } + } + return new Float32Array(nhwcData); +} + /** * 将nhwc排布数据转为nchw排布数据 * @param {Array} data tensor data diff --git a/packages/paddlejs-core/src/runner.ts b/packages/paddlejs-core/src/runner.ts index d9eca95d..c7f08ed7 100644 --- a/packages/paddlejs-core/src/runner.ts +++ b/packages/paddlejs-core/src/runner.ts @@ -15,7 +15,7 @@ import env from './env'; import type OpExecutor from './opFactory/opExecutor'; -import { accShape } from './opFactory/utils'; +import { accShape, genTensorData } from './opFactory/utils'; import postOpsList from './postOps'; export default class Runner { @@ -47,7 +47,7 @@ export default class Runner { } } - async init() { + async init(modelJSON?: Model) { if (!GLOBALS.backendInstance) { console.error('ERROR: Haven\'t register backend'); return; @@ -59,7 +59,7 @@ export default class Runner { } else { await GLOBALS.backendInstance.init(); - await this.load(); + await this.load(modelJSON); } this.genFeedData(); this.genGraph(); @@ -77,11 +77,11 @@ export default class Runner { } } - async load() { + async load(modelJSON?: Model) { const { modelPath, modelObj = null } = this.runnerConfig; if (modelPath) { const loader = new Loader(modelPath); - this.model = await loader.load(); + this.model = await loader.load(modelJSON); } else if (modelObj?.model && modelObj?.params) { const { @@ -126,9 +126,11 @@ export default class Runner { } async checkModelLoaded() { + if (!this.model) { + await this.load(); + } if (this.weightMap.length === 0) { console.info('It\'s better to preheat the model before running.'); - await this.load(); this.genFeedData(); this.genGraph(); this.genOpData(); @@ -168,7 +170,12 @@ export default class Runner { return callback ? callback(result) : result; } - async predictWithFeed(data: number[] | InputFeed[] | ImageData, callback?, shape?: number[]) { + async predictWithFeed( + data: number[] | InputFeed[] | ImageData, + callback?, + shape?: number[], + isNchw?: boolean + ) { const { fc = 3, fw, fh } = this.feedShape; let inputFeed; @@ -194,6 +201,16 @@ export default class Runner { ]; } } + else if (ArrayBuffer.isView(data)) { + inputFeed = [ + { + data, + shape: shape || [1, fc, fh, fw], + name: 'image', + persistable: true + } + ]; + } else { // 类imageData类型 const { width, height, data: inputData } = data as ImageData; @@ -207,6 +224,16 @@ export default class Runner { ]; } + if (!isNchw) { + const feed = inputFeed[0]; + feed.data = genTensorData( + feed.data, + 'nhwc', + feed.shape, + false + ); + } + let result = []; if (env.get('backend') === 'wasm') { await GLOBALS.backendInstance.predict(inputFeed[0].data, this.model.index); @@ -226,7 +253,7 @@ export default class Runner { const { type, feedShape, webglFeedProcess } = this.runnerConfig; this.feedShape = this.model.feedShape || feedShape; const { fc = 3, fh, fw } = this.feedShape; - const vars = this.model.vars; + const { vars, feed } = this.model; let preheatFeedData; if (type === GraphType.MultipleInput) { // 默认第1个是输入op, 形为inputs: {X: [a, b]} @@ -244,11 +271,11 @@ export default class Runner { } } else { + preheatFeedData = feed || findVarByKey(vars, 'image'); const feedC = env.get('backend') !== 'wasm' && webglFeedProcess ? 4 : fc; - preheatFeedData = findVarByKey(vars, 'image'); const imageBaseInfo = { name: 'image', - shape: [1, feedC, fh, fw] + shape: [1, fc, fh, fw] }; preheatFeedData = Object.assign( imageBaseInfo, @@ -258,6 +285,10 @@ export default class Runner { persistable: true } ); + + if (env.get('backend') !== 'wasm' && webglFeedProcess) { + preheatFeedData.shape[1] = 4; + } } AddItemToVars(vars, preheatFeedData); @@ -265,50 +296,53 @@ export default class Runner { updateFeedData(inputFeed) { const feed = inputFeed[0]; - const imageOp = this.weightMap.find(item => { + const imageOps = this.weightMap.filter(item => { if (!item.opData) { return null; } const tensorData = item.opData.inputTensors; return tensorData.find(tensor => tensor.tensorId.endsWith('_image')); - }) as OpExecutor; - - const imageInputTensor = imageOp.opData.inputTensors.find( - tensor => tensor.tensorId.endsWith('_image') - ); - imageInputTensor.data = feed.data; - - const { - webglFeedProcess = false, - keepRatio = true - } = this.runnerConfig; - if (webglFeedProcess || env.get('webgl_gpu_pipeline')) { - // support imageDataLike feed which has unit8ClampedArray data and width + height or shape - // support ImageElementLike feed which is HTMLImageElement or HTMLVideoElement or HTMLCanvasElement - let shape = feed.shape || [1, 1, feed.height, feed.width]; - let feedData = new Uint8Array(feed.data || []); - const isImageElementLike = feed.width && feed.height && !feed.data; - if (isImageElementLike) { - const w = (feed as HTMLImageElement).naturalWidth || feed.width; - const h = (feed as HTMLImageElement).naturalHeight || feed.height; - shape = [1, 1, h, w]; - feedData = feed; + }) as OpExecutor[]; + + for (const imageOp of imageOps) { + for (const tensor of imageOp.opData.inputTensors) { + if (tensor.tensorId.endsWith('_image')) { + tensor.data = feed.data; + } } - const imageInputTensorParams = imageInputTensor.opts; - imageInputTensorParams.shape = shape; - const imageOpData = imageOp.opData; - const imageTensor = new Tensor(imageInputTensorParams); - imageTensor.data = feedData; - imageOpData.inputTensors = [imageTensor]; + const { + webglFeedProcess = false, + keepRatio = true + } = this.runnerConfig; + if (webglFeedProcess || env.get('webgl_gpu_pipeline')) { + const inputTensor = imageOp.opData.inputTensors.find(tensor => tensor.tensorId.endsWith('_image')); + // support imageDataLike feed which has unit8ClampedArray data and width + height or shape + // support ImageElementLike feed which is HTMLImageElement or HTMLVideoElement or HTMLCanvasElement + let shape = feed.shape || [1, 1, feed.height, feed.width]; + let feedData = new Uint8Array(feed.data || []); + const isImageElementLike = feed.width && feed.height && !feed.data; + if (isImageElementLike) { + const w = (feed as HTMLImageElement).naturalWidth || feed.width; + const h = (feed as HTMLImageElement).naturalHeight || feed.height; + shape = [1, 1, h, w]; + feedData = feed; + } - const [h, w] = shape.slice(-2); - const [dh, dw] = imageOpData.outputTensors[0].shape.slice(-2); - const scale = this.mediaProcessor.cover(w, h, dw, dh); - imageOp.uniform.u_scale.value = scale; - imageOp.uniform.u_keep_ratio.value = +keepRatio; + const imageInputTensorParams = inputTensor.opts; + imageInputTensorParams.shape = shape; + const imageOpData = imageOp.opData; + const imageTensor = new Tensor(imageInputTensorParams); + imageTensor.data = feedData; + imageOpData.inputTensors = [imageTensor]; + + const [h, w] = shape.slice(-2); + const [dh, dw] = imageOpData.outputTensors[0].shape.slice(-2); + const scale = this.mediaProcessor.cover(w, h, dw, dh); + imageOp.uniform.u_scale.value = scale; + imageOp.uniform.u_keep_ratio.value = +keepRatio; + } } - } async execute() { @@ -419,4 +453,16 @@ export default class Runner { stopPredict() { this.isPaused = true; } + + dispose() { + for (const item of this.weightMap) { + if (!item.opData) { + continue; + } + const opData = item.opData; + GLOBALS.backendInstance.dispose && GLOBALS.backendInstance.dispose(opData); + } + this.weightMap = []; + this.model = null; + } } diff --git a/packages/paddlejs-core/src/transform/feedProcess.ts b/packages/paddlejs-core/src/transform/feedProcess.ts index fdd78dba..307dc55d 100644 --- a/packages/paddlejs-core/src/transform/feedProcess.ts +++ b/packages/paddlejs-core/src/transform/feedProcess.ts @@ -46,16 +46,19 @@ export default class WebglFeedProcess extends Transformer { AddItemToVars(vars, [originImgVar, processImgVar]); // change recieve_img op input - const imageOriginInputOp = ops.find(item => { + const imageOriginInputOps = ops.filter(item => { const inputsMap = item.inputs; return Object.keys(inputsMap).find(key => inputsMap[key][0] === 'image'); }); - const inputs = imageOriginInputOp.inputs; - Object.keys(inputs).forEach(key => { - if (inputs[key][0] === 'image') { - inputs[key][0] = IMG_PRE_PROCESS_VAR; - } - }); + + for (const imageOriginInputOp of imageOriginInputOps) { + const inputs = imageOriginInputOp.inputs; + Object.keys(inputs).forEach(key => { + if (inputs[key][0] === 'image') { + inputs[key][0] = IMG_PRE_PROCESS_VAR; + } + }); + } // make feed post process op const imgPreProcessOp: ModelOp = {