github-featured/alibaba--mnn
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

MNN:Sync: Sync Internal 3.3.0

Browse Source
This commit is contained in:
xiaying
2025-10-31 10:31:53 +08:00
parent a8fb0de0b7
commit 1ca62a02f7
243 changed files with 27422 additions and 13364 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+34 -23
View File
@@ -82,15 +82,15 @@ option(MNN_BUILD_AUDIO "Build audio api in MNN." OFF)
option(MNN_SME2 "Use Arm sme2 instructions" ON)
if (MNN_BUILD_MINI)
set(MNN_SKIPBUILD_GEOMETRY ON)
set(MNN_REDUCE_SIZE ON)
set(MNN_SKIPBUILD_GEOMETRY ON CACHE BOOL "<docstring>" FORCE)
set(MNN_REDUCE_SIZE ON CACHE BOOL "<docstring>" FORCE)
endif()
if (MNN_REDUCE_SIZE)
set(MNN_SUPPORT_DEPRECATED_OP OFF)
set(MNN_SUPPORT_DEPRECATED_OPV2 OFF)
set(MNN_SUPPORT_QUANT_EXTEND OFF)
set(MNN_USE_SPARSE_COMPUTE OFF)
set(MNN_SUPPORT_DEPRECATED_OP OFF CACHE BOOL "<docstring>" FORCE)
set(MNN_SUPPORT_DEPRECATED_OPV2 OFF CACHE BOOL "<docstring>" FORCE)
set(MNN_SUPPORT_QUANT_EXTEND OFF CACHE BOOL "<docstring>" FORCE)
set(MNN_USE_SPARSE_COMPUTE OFF CACHE BOOL "<docstring>" FORCE)
endif()
IF (OHOS AND MNN_INTERNAL)
@@ -106,18 +106,19 @@ IF (NOT DEFINED MNN_USE_SPARSE_COMPUTE)
ENDIF()
IF (MNN_BUILD_LLM)
set(MNN_LOW_MEMORY ON)
set(MNN_SUPPORT_TRANSFORMER_FUSE ON)
set(MNN_LOW_MEMORY ON CACHE BOOL "<docstring>" FORCE)
set(MNN_SUPPORT_TRANSFORMER_FUSE ON CACHE BOOL "<docstring>" FORCE)
IF (MNN_BUILD_LLM_OMNI)
set(MNN_BUILD_OPENCV ON)
set(MNN_BUILD_AUDIO ON)
set(MNN_BUILD_OPENCV ON CACHE BOOL "<docstring>" FORCE)
set(MNN_BUILD_AUDIO ON CACHE BOOL "<docstring>" FORCE)
set(MNN_IMGCODECS ON CACHE BOOL "<docstring>" FORCE)
ENDIF()
ENDIF()
IF (MNN_BUILD_DIFFUSION)
set(MNN_LOW_MEMORY ON)
set(MNN_SUPPORT_TRANSFORMER_FUSE ON)
set(MNN_BUILD_OPENCV ON)
set(MNN_LOW_MEMORY ON CACHE BOOL "<docstring>" FORCE)
set(MNN_SUPPORT_TRANSFORMER_FUSE ON CACHE BOOL "<docstring>" FORCE)
set(MNN_BUILD_OPENCV ON CACHE BOOL "<docstring>" FORCE)
ENDIF()
IF(NOT MNN_BUILD_SHARED_LIBS AND MNN_SEP_BUILD)
@@ -656,6 +657,15 @@ IF(MNN_QNN)
list(APPEND MNN_OBJECTS_TO_LINK $<TARGET_OBJECTS:MNN_QNN>)
ENDIF()
# NEUROPILOT
IF(MNN_NEUROPILOT)
target_compile_options(MNNCore PRIVATE -DMNN_NEUROPILOT=1)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/source/backend/neuropilot)
list(APPEND MNN_OBJECTS_TO_LINK $<TARGET_OBJECTS:MNN_NEUROPILOT>)
ENDIF()
# Vulkan
# Vulkan
IF(MNN_VULKAN)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/source/backend/vulkan/)
@@ -807,6 +817,16 @@ ELSE()
ENDIF()
ENDIF()
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/tools/audio)
IF(MNN_BUILD_AUDIO)
IF(MNN_SEP_BUILD)
list(APPEND MNN_DEPS MNNAudio)
ELSE()
list(APPEND MNN_TARGETS MNNAudio)
list(APPEND MNN_OBJECTS_TO_LINK $<TARGET_OBJECTS:MNNAudio>)
ENDIF()
ENDIF()
IF(MNN_BUILD_LLM)
include(${CMAKE_CURRENT_LIST_DIR}/transformers/llm/engine/CMakeLists.txt)
IF(NOT MNN_SEP_BUILD)
@@ -887,21 +907,12 @@ IF(WIN32 AND MNN_BUILD_CONVERTER AND MNN_BUILD_SHARED_LIBS)
ENDIF()
# Merge MNN/MNNExpress/MNNOpenCV and other backends into one .lib/.dll on Windows
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/tools/audio)
IF(MNN_BUILD_AUDIO AND NOT MNN_SEP_BUILD)
IF(MSVC)
target_compile_definitions(MNNAudio PRIVATE "-DBUILDING_MNN_DLL" INTERFACE "-DUSING_MNN_DLL")
ENDIF()
target_sources(MNN PRIVATE $<TARGET_OBJECTS:MNNAudio>)
ENDIF()
if(CMAKE_SYSTEM_NAME MATCHES "^Linux")
# Using -pthread, needed by thread-safe implemention of glibc, is better than only using -lpthread
# https://stackoverflow.com/questions/23250863/difference-between-pthread-and-lpthread-while-compiling
target_link_libraries(MNN PUBLIC -pthread dl)
elseif(CMAKE_SYSTEM_NAME MATCHES "^Android")
target_link_libraries(MNN PUBLIC log m)
target_link_libraries(MNN PUBLIC log m android)
else()
endif()
if (NOT MNN_BUILD_SHARED_LIBS)
backupcode/cpubackend/compute/DeconvolutionWithStride.cpp
+3 -3
View File
@@ -70,7 +70,7 @@ static void _winograd(const DeconvolutionWithStride::ComputeUnit& unit, int thre
el[2] = 0;
el[3] = 0;
size_t parameters[6];
parameters[0] = eP * sizeof(float);
parameters[0] = eP * lP * sizeof(float);
parameters[1] = ROUND_UP(ic, lP);
parameters[2] = oc;
parameters[3] = eP * 4 * sizeof(float);
@@ -129,7 +129,7 @@ static void _gemmAndIm2col(const DeconvolutionWithStride::ComputeUnit& unit, int
el[2] = 0;
el[3] = 0;
size_t parameters[6];
parameters[0] = eP * sizeof(float);
parameters[0] = eP * lP * sizeof(float);
parameters[1] = ROUND_UP(ic, lP);
parameters[2] = oc;
parameters[3] = eP * 4 * sizeof(float);
@@ -272,7 +272,7 @@ void DeconvolutionWithStride::_extract(const Op* convOp) {
std::shared_ptr<ConvolutionCommon::Int8Common> quanCommon;
ConvolutionCommon::getConvParameters(&quanCommon, backend(), convOp, &tempWeight, &tempWeightSize);
srcCount = tempWeightSize / kx / ky / outputCount;
std::shared_ptr<Tensor> weightWrap(
Tensor::create<float>(std::vector<int>{srcCount, outputCount, ky * kx}, (void*)tempWeight));
demo/exec/pictureRecognition_module.cpp
+2 -2
View File
@@ -217,7 +217,7 @@ int main(int argc, const char* argv[]) {
// Create Input
int batchSize = argc - 3;
auto input = MNN::Express::_Input({batchSize, 3, width, height}, MNN::Express::NC4HW4);
auto input = MNN::Express::_Input({batchSize, 3, height, width}, MNN::Express::NC4HW4);
for (int batch = 0; batch < batchSize; ++batch) {
int size_w = width;
int size_h = height;
@@ -257,4 +257,4 @@ int main(int argc, const char* argv[]) {
rtmgr->updateCache();
return 0;
}
}
docs/compile/cmake.md
+1
View File
@@ -59,6 +59,7 @@ MNN使用CMake构建项目，CMake中的宏定义列表如下：
| MNN_QNN | 是否构建`QNN`后端，默认为`OFF` |
| MNN_QNN_ONLINE_FINALIZE | 在`MNN_QNN`开启的基础上,是否构建在线编译模式的QNN后端，默认为`ON` |
| MNN_QNN_CONVERT_MODE | 在`MNN_QNN`开启的基础上,是否构建Convert模式的QNN后端，默认为`OFF` |
| MNN_NEUROPILOT | 是否构建MLA的`NPU`离线转换后端或执行插件，默认为`OFF` |
| MNN_NPU | 是否构建HIAI的`NPU`后端，默认为`OFF` |
| MNN_USE_SPARSE_COMPUTE | 是否使用稀疏计算，默认为`ON` |
| MNN_BUILD_BENCHMARK | 是否构建MNN的性能测试，默认为`OFF` |
docs/compile/other.md
+2
View File
@@ -63,6 +63,7 @@
- `llm_demo` 大语言模型推理示例程序
- `diffusion_demo` 扩散模型示例程序
- `llm_bench` 大语言模型测评工具
- `quantize_llm` 大语言模型feature map量化工具
## 测试工具
- 相关编译选项
- `MNN_BUILD_TOOLS` 是否编译测试工具
@@ -93,6 +94,7 @@
- `fuseTest` 测试 GPU 自定义算子的功能，目前仅支持 Vulkan Buffer 模式
- `GpuInterTest.out` 测试 GPU 内存输入的功能，目前仅支持 OpenCL Buffer 模式与 OpenGL texture 模式，编译时需打开 MNN_OPENCL 与 MNN_OPENGL
- `LoRA` 将LorA权重添加到模型权重中
- `compilefornpu` 将Npu要运行的部分转换为Plugin算子
## Benchmark工具
- 相关编译选项
- `MNN_BUILD_BENCHMARK` 是否编译Benchmark工具
docs/inference/npu.md
+109 -43
View File
@@ -6,63 +6,129 @@
- NNAPI
- HIAI
目前NPU相关后端均不支持可变形状、控制流等动态模型，算子数相比CPU/GPU支持要少，建议根据NPU是否能跑通，反复调整模型结构。
同时，由于QNN、CoreML与NNAPI在MNN中共用同一个Backend Type，这三个后端对应的编译宏MNN_QNN、MNN_COREML、MNN_NNAPI在编译时，至多只能打开一个。
## QNN
适用于使用高通芯片且配备高通Hexagon张量处理器（Hexagon Tensor Processor，HTP）的机型，可参考[高通官网的设备支持列表](https://docs.qualcomm.com/bundle/publicresource/topics/80-63442-50/overview.html#supported-snapdragon-devices)。
### 获得QNN依赖
QNN后端依赖QNN SDK中的`/include/QNN`与`lib`，首先，我们需要获得相关依赖。
### QNN后端整体介绍
- MNN通过调用QNN SDK的CPP API构建了MNN-QNN后端，以期在能够使用高通NPU的设备上取得推理加速。
- 我们支持了两种运行模式：
- 在线构图模式，在线编译和序列化QNN计算图。
- 支持静态形状的常规模型的推理。
- 离线构图模式则先借助MNN的离线工具缓存QNN计算图的序列化产物，接着在运行时直接读取产物，可以节省初始化时间。
- 支持静态形状/有限形状组合的常规模型的推理。
- 可支持部分llm模型的推理加速。
### 准备工作
#### 开发环境
- Host
- 在线构图模式：无要求。
- 离线构图模式：一台x86_64，Linux的机器（链路中的部分QNN工具必须在此环境中运行）。
- Device
- 一台可以使用高通NPU的设备；为便于陈述，下文假设这是一台Android系统的设备。
#### 明确硬件架构
QNN后端的部分使用步骤（如生成离线产物，确定QNN的NPU库依赖等）需要指定device的硬件架构对应的SOC ID以及HEXAGON ARCH。对于一些常见的硬件架构，我们列举如下供你参考：
| 硬件 | SOC ID | HEXAGON ARCH |
| :------ | :----- | :----------- |
| 8 Gen 1 | 36 | 69 |
| 8 Gen 2 | 43 | 73 |
| 8 Gen 3 | 57 | 75 |
| 8 Elite | 69 | 79 |
对于其他的硬件架构，你可以参考高通官网的设备支持列表。
#### 获得QNN依赖
MNN-QNN后端依赖QNN SDK中的`include/QNN`与`lib`，可通过以下步骤获取依赖：
- [注册高通账号](https://myaccount.qualcomm.com/signup)
- 访问Qualcomm AI Engine Direct SDK（即QNN SDK）[官网](https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk)，下载SDK。
- 参考以下指令，将下载的sdk中的`/include/QNN`与`lib`拷贝到MNN源码中的对应位置。
- 访问Qualcomm AI Engine Direct SDK（即QNN SDK），下载SDK，并解压。比如`/home/xiaying/third/qnn/qairt/2.38.0.250901`
- 修改`~/.bashrc` ，增加SDK路径到环境变量, 然后运行 `source ~/.bashrc` 或者重启终端。eg：
```
QNN_SDK_ROOT="/YOUR/QNN/SDK/PATH" # modify this variable according to your environment
MNN_ROOT="/YOUR/MNN/PATH" # modify this variable according to your environment
INCLUDE_SRC="${QNN_SDK_ROOT}/include/QNN"
LIB_SRC="${QNN_SDK_ROOT}/lib"
INCLUDE_DEST="${MNN_ROOT}/source/backend/qnn/3rdParty/include"
LIB_DEST="${MNN_ROOT}/source/backend/qnn/3rdParty/lib"
mkdir "${MNN_ROOT}/source/backend/qnn/3rdParty"
cp -r ${INCLUDE_SRC} ${INCLUDE_DEST}
cp -r ${LIB_SRC} ${LIB_DEST}
export QNN_SDK_ROOT=/home/xiaying/third/qnn/qairt/2.38.0.250901
export QNN_ROOT=/home/xiaying/third/qnn/qairt/2.38.0.250901
export HEXAGON_SDK_ROOT=/home/xiaying/third/qnn/qairt/2.38.0.250901
```
### QNN后端编译
- 编译 MNN 时打开编译宏`MNN_QNN`，即`-DMNN_QNN=ON`。
- 如果运行离线编译QNN模型(离线编译方法：使用MNN2QNNModel工具)，需要开启`MNN_WITH_PLUGIN`宏。若需要减小库体积，可以选择关闭`MNN_QNN_ONLINE_FINALIZE`宏
### 在线构图模式，推理常规模型
在线构图模式的使用步骤与其他后端基本一致，主要包含以下三部分。
#### Host，交叉编译Device侧的MNN库及AI应用程序
- 参考[“主库编译”](../compile/engine.md#主库编译)，配置Android系统的编译环境及CMake变量。
- 添加额外的CMake变量并编译：`-DMNN_QNN=ON`、`-DMNN_QNN_CONVERT_MODE=OFF`、`-DMNN_WITH_PLUGIN=OFF`。
### QNN后端运行
- Backend Type设置为`MNN_FORWARD_NN`，即 5 。
- 除MNN相关的库之外，QNN后端在运行时还依赖四个QNN库，可参考以下指令，将QNN中的库拷贝到设备中。其中变量`HEXAGON_ARCH`需要与你的目标机型匹配，可参考[高通官网的设备支持列表](https://docs.qualcomm.com/bundle/publicresource/topics/80-63442-50/overview.html#supported-snapdragon-devices)，如8gen3的设备，需要设定`HEXAGON_ARCH="75"`。
#### 推送资源至Device
参考下面的指令，将以下资源推送到Device侧
- AI应用程序。
- 交叉编译得到的Device侧的MNN库。
- QNN库（`libQnnHtp.so`、`libQnnHtpV${HEXAGON_ARCH}Stub.so`、`libQnnHtpV${HEXAGON_ARCH}Skel.so`、`libQnnHtpPrepare.so`）。
- MNN模型。
```
HEXAGON_ARCH="75" # modify this variable according to your environment
MNN_ROOT="/YOUR/MNN/PATH" # modify this variable according to your environment
ANDROID_PATH="/data/local/tmp"
adb push ${MNN_ROOT}/source/backend/qnn/3rdParty/lib/aarch64-android/libQnnHtp.so ${ANDROID_PATH}/libQnnHtp.so
MNN_ROOT_PATH="/YOUR/MNN/ROOT/PATH" # modify this variable according to your environment
BUILD_ANDROID_PATH="/your/build/andorid/path" # modify this variable according to your environment
ANDROID_WORKING_DIR="/data/local/tmp" # modify this variable according to your environment
/*
如下libQnnHtpPrepare.so和libQnnSystem.so两个库，根据情况二选一
- 如果在线生成qnn图模型，运行时需要libQnnHtpPrepare.so
- 如果离线生成qnn图模型，运行时需要libQnnSystem.so
*/
adb push ${MNN_ROOT}/source/backend/qnn/3rdParty/lib/aarch64-android/libQnnHtpPrepare.so ${ANDROID_PATH}/libQnnHtpPrepare.so
adb push ${MNN_ROOT}/source/backend/qnn/3rdParty/lib/aarch64-android/libQnnSystem.so ${ANDROID_PATH}/libQnnSystem.so
# push mnn libs
cd ${BUILD_ANDROID_PATH}
find . -name "*.so" | while read solib; do
adb push $solib ${ANDROID_WORKING_DIR}
done
cd -
adb push ${MNN_ROOT}/source/backend/qnn/3rdParty/lib/aarch64-android/libQnnHtpV${HEXAGON_ARCH}Stub.so ${ANDROID_PATH}/libQnnHtpV${HEXAGON_ARCH}Stub.so
adb push ${MNN_ROOT}/source/backend/qnn/3rdParty/lib/hexagon-v${HEXAGON_ARCH}/unsigned/libQnnHtpV${HEXAGON_ARCH}Skel.so ${ANDROID_PATH}/libQnnHtpV${HEXAGON_ARCH}Skel.so
```
- 为了动态链接到QNN HTP相关的库，需要在环境变量`ADSP_LIBRARY_PATH`中添加QNN HTP库所在的目录（部分机型上有效）。如果这样也没法成功链接，可将可执行文件push到QNN HTP库所在目录（如`/data/local/tmp`），cd到对应目录后，再运行可执行文件，参考如下指令。
```
adb shell "cd /data/local/tmp && LD_LIBRARY_PATH=/data/local/tmp ADSP_LIBRARY_PATH=/data/local/tmp ./MyExe.out"
# push your AI exe
adb push /your/AI/exe ${ANDROID_WORKING_DIR}
# push QNN libs
adb push ${QNN_SDK_ROOT}/lib/aarch64-android/libQnnHtp.so ${ANDROID_WORKING_DIR}
adb push ${QNN_SDK_ROOT}/lib/aarch64-android/libQnnHtpV${HEXAGON_ARCH}Stub.so ${ANDROID_WORKING_DIR}
adb push ${QNN_SDK_ROOT}/lib/hexagon-v${HEXAGON_ARCH}/unsigned/libQnnHtpV${HEXAGON_ARCH}Skel.so ${ANDROID_WORKING_DIR}
# The following lib is only needed in the online case.
adb push ${QNN_SDK_ROOT}/lib/aarch64-android/libQnnHtpPrepare.so ${ANDROID_WORKING_DIR}
# push MNN models
adb push model.mnn ${ANDROID_WORKING_DIR}
```
### QNN量化功能说明
- 仅权重量化（激活是浮点）：只支持Linear权重int8、channel-wise的对称量化。
- 激活&权重都量化：支持激活per-tensor对称量化，权重是int8/int4、channel-wise的对称量化。
#### Device，链接并运行
- 链接QNN库
- 为了动态链接到QNN HTP相关的库，需要在环境变量`ADSP_LIBRARY_PATH`中添加QNN HTP库所在的目录（部分机型上有效）。如果这样也没法成功链接，可将可执行文件，QNN HTP库推送至同一目录，cd到对应目录后，再运行可执行文件，参考如下指令。
```
adb shell "cd ${ANDROID_WORKING_DIR} && export LD_LIBRARY_PATH=.:${ANDROID_LD_LIBRARY_PATH} && export ADSP_LIBRARY_PATH=.:${ANDROID_ADSP_LIBRARY_PATH} && ./your/mnn/qnn/ai/exe"
```
- 配置MNN
- Backend Type设置为`MNN_FORWARD_NN`，即5。
- 在使用Module API推理时，需要设定`Module::Config`中的`shapeMutable`字段为`false`。
### 离线构图模式，推理常规模型
相较于在线构图模式，离线构图模式额外包含一次编译（构建生成离线产物需要的MNN库）以及一个模型转换步骤（将原始的MNN模型转化成QNN产物），具体如下。
#### Host，编译生成离线模式产物需要的的MNN库及相应MNN离线工具
- 添加额外的CMake变量并编译：`-DMNN_QNN=ON`、`-DMNN_QNN_CONVERT_MODE=ON`、`-DMNN_WITH_PLUGIN=OFF`、`-DMNN_BUILD_TOOLS=ON`。
#### Host，生成QNN离线构图产物
调用`MNN2QNNModel`工具，针对Device的硬件架构，生成QNN离线产物（`model_${SOC_ID}_${HEXAGON_ARCH}.bin`）以及替代模型（`model_${SOC_ID}_${HEXAGON_ARCH}.mnn`），具体可参考[该工具的用法](../tools/convert.md#mnn2qnnmodel)。
#### Host，交叉编译Device侧的MNN库及AI应用程序
- 参考[“主库编译”](../compile/engine.md#主库编译)，配置Android系统的编译环境及CMake变量。
- 添加额外的CMake变量并编译：`-DMNN_QNN=ON`、`-DMNN_QNN_CONVERT_MODE=OFF`、`-DMNN_WITH_PLUGIN=ON`。
#### 推送资源至Device
与[在线构图模式的情况](#推送资源至device)类似，但有以下两点不同：
- 依赖的QNN库变为`libQnnHtp.so`、`libQnnHtpV${HEXAGON_ARCH}Stub.so`、`libQnnHtpV${HEXAGON_ARCH}Skel.so`、`libQnnSystem.so`（不再依赖`libQnnHtpPrepare.so`，而是依赖`libQnnSystem.so`）。
- 不再使用原始的MNN模型，而是需要QNN离线产物（`model_${SOC_ID}_${HEXAGON_ARCH}.bin`）以及替代模型（`model_${SOC_ID}_${HEXAGON_ARCH}.mnn`）。
#### Device，链接并运行
- 配置MNN
- 指定backend type为0（CPU）。读取并推理QNN离线产物的功能被封装在Plugin算子内，该算子被注册在CPU后端，因此，此时需要指定backend type为CPU。
- 在Device侧，如果你的离线产物和你的应用的工作目录不一致，那么你需要在程序中通过`Executor::RuntimeManager::setExternalPath`接口设定离线产物所在的目录。
- 链接QNN库
- 离线构图模式对于链接的要求和在线构图模式一致。
## CoreML
适用于 Mac / iOS / iPad
docs/tools/convert.md
+56 -72
View File
@@ -334,84 +334,68 @@ cat mobilenet_v1.json
## MNN2QNNModel
### 功能
利用QNN工具将mnn模型转为可以在QNN运行的mnn模型结构文件以及QNN离线序列化模型，后续可以在QNN上运行该离线模型。
- 注意：该工具目前仅支持在Linux环境(推荐Ubuntu22.04)运行，需要提前下载QNN SDK，参考QNN环境准备(docs/inference/npu.md)
### 参数
`Usage: ./MNN2QNNModel src.mnn dst.mnn qnn_sdk_path qnn_model_name qnn_context_config.json input_type_size input_type_0/input_type_mnn_0 input_type_1/input_type_mnn_1`
- `src.mnn:str` 源mnn模型文件路径
- `dst.mnn:str` 目标mnn模型文件路径
- `qnn_sdk_path:str` QNN SDK绝对路径,建议2.37及之后的版本
- `qnn_model_name:str` 转完后的QNN模型图名字，同时需要新建同名文件夹，后续生成的QNN产物放在该目录下
- `qnn_context_config.json:str` QNN生成context binary的配置文件（示例文件：source/backend/qnn/convertor/config_example/context_config.json和source/backend/qnn/convertor/config_example/htp_backend_extensions.json），通常需要改context_config.json文件中路径地址，htp_backend_extensions.json中graph_names（需要与qnn_model_name保持一致）、soc_id、dsp_arch（根据机型参考[高通官网的设备架构表](https://docs.qualcomm.com/bundle/publicresource/topics/80-63442-50/overview.html#supported-snapdragon-devices)进行设置）
- （可选）`input_type_size:str`，对于要指定输入形状或者多种输入形状的场景，设置输入形状的种类数。例如想生成2种类型的输入形状，则设置2，后面两个参数则需要传入2中输入形状。
- 说明：输入形状设置可以是:input_type_x直接设置shape字符串，或者是input_type_mnn设置输入的mnn文件。通常建议单输入单输出模型采用直接设置字符串形状较方便，多输入多输出模型通过mnn模型方式更容易避免设置出错。
- （可选）`input_type_x`说明：单输入模型形状设置例如'1x3x512x512'。多输入模型形状设置每个输入形状使用'_'连接，例如'1x3x512x512_1x128'。多输入多输出模型建议使用input.mnn为输入形状设置。
### 使用示例
```
cd mnn_path
mkdir build
cd build
// 确保已经把高通SDK头文件拷贝到对应路径
cmake .. -DMNN_QNN=ON -DMNN_QNN_CONVERT_MODE=ON -DMNN_SUPPORT_TRANSFORMER_FUSE=ON
make -j16
```
#### 默认输入形状(不手动设置输入形状)
```
./MNN2QNNModel mnnfuse_smolvlm/visual.mnn qnn_smolvlm_model.mnn /mnt/2Tpartition/tianbu/QNN/qairt/2.37.0.250724 qnn_smolvlm_model ../source/backend/qnn/convertor/config_example/context_config.json
该工具针对特定的高通硬件架构，为原始的MNN模型生成MNN-QNN后端需要的替代模型以及QNN离线产物。目前，支持静态形状的模型以及有限输入形状组合的模型。
### 运行环境要求
该工具必须在 x86_64 架构的 Linux 系统上运行（部分QNN SDK中的离线工具必须在此环境中运行）。
### 编译
添加额外的CMAKE变量并编译：`-DMNN_QNN=ON -DMNN_QNN_CONVERT_MODE=ON -DMNN_WITH_PLUGIN=OFF -DMNN_BUILD_TOOLS=ON -DMNN_SUPPORT_TRANSFORMER_FUSE=ON`。
### 用法说明
该工具的用法如下
Can't open file:/sys/devices/system/cpu/cpufreq/schedutil/affected_cpus
Can't open file:/sys/devices/system/cpu/cpufreq/boost/affected_cpus
CPU Group: [ 20 21 13 23 1 15 3 17 5 19 7 10 11 9 12 22 0 14 2 16 4 18 6 8 ], 2200000 - 3800000
The device supports: i8sdot:0, fp16:0, i8mm: 0, sve2: 0, sme2: 0
Load Cache file error.
2025-07-30 16:10:05,068 - INFO - qnn-model-lib-generator: Model cpp file path : qnn_smolvlm_model/qnn_smolvlm_model.cpp
2025-07-30 16:10:05,068 - INFO - qnn-model-lib-generator: Model bin file path : qnn_smolvlm_model/qnn_smolvlm_model.bin
2025-07-30 16:10:05,069 - INFO - qnn-model-lib-generator: Library target : [['x86_64-linux-clang']]
2025-07-30 16:10:05,069 - INFO - qnn-model-lib-generator: Library name : qnn_smolvlm_model
2025-07-30 16:10:05,069 - INFO - qnn-model-lib-generator: Output directory : qnn_smolvlm_model/lib
2025-07-30 16:10:05,069 - INFO - qnn-model-lib-generator: Output library name : qnn_smolvlm_model
2025-07-30 16:10:59,923 - INFO - qnn-model-lib-generator: Target: x86_64-linux-clang Library: /home/mnnteam/tianbu/AliNNPrivate/build/qnn_smolvlm_model/lib/x86_64-linux-clang/libqnn_smolvlm_model.so
[Pass]: qnn-model-lib-generator success!
qnn-context-binary-generator pid:1490535
[Pass]: qnn-context-binary-generator success!
npu model path:./qnn_smolvlm_model.bin
[All Pass]: npu model generator success!
```
`[All Pass]: npu model generator success!`说明整个过程成功。
#### 推理方式说明
- 生成所需的两个模型dst.mnn和qnn_model_name/binary/qnn_model_name.bin两个QNN文件。
- 将这两个文件替换原来src.mnn使用
- 需要打开`MNN_WITH_PLUGIN`宏重新编译，运行后端设置为CPU
- 正确性验证，例如：
```
/*
1、确保已经把高通库文件push到对应路径，已经环境变量设置。参考QNN环境准备(docs/inference/npu.md)
2、shapeMutable设为false，在input.json文件中设置
3、需要设置CPU后端运行，实际QNN图以Plugin插件形式运行在QNN后端。
*/
./ModuleBasic.out qnn_smolvlm_model.mnn dir 0 0 10
./MNN2QNNModel <qnnSDKPath> <socId> <hexagonArch> <srcMNNPath> <outputDir> [totalShapeNum] [inputShape1] [inputShape2] ...
```
#### 自定义输入形状(多种推理尺寸设置)
```
./MNN2QNNModel /home/mnnteam/tianbu/models/mnnfuse_fastvlm2_q8b0sym/visual.mnn ./qnn_models/fastvlm_visual_8_sym_57_v75.mnn /mnt/2Tpartition/tianbu/QNN/qairt/2.37.0.250724/ fastvlm_visual_8_sym_57_v75 ../source/backend/qnn/convertor/config_example/context_config.json 2 1x3x512x512 1x3x1024x1024
参数配置说明如下：
| 参数 | 说明 | 是否必须 |
| :--- | :--- | :--- |
| `<qnnSDKPath>` | QNN SDK 的根目录路径。 | 是 |
| `<socId>` | 目标 SoC 的 ID。常用 ID 参考：8Gen2 -> `43`, 8Gen3 -> `57`, 8 Elite -> `69`。其他型号请参考高通官方文档。 | 是 |
| `<hexagonArch>` | Hexagon架构版本。常用架构参考：8Gen2 -> `73`, 8Gen3 -> `75`, 8 Elite -> `79`。其他型号请参考高通官方文档。 | 是 |
| `<srcMNNPath>` | 待转换的原始 MNN 模型文件路径（`.mnn` 文件）。 | 是 |
| `<outputDir>` | 用于存放生成产物的目录。工具会在此目录下生成一个新的 `.mnn` 文件（替代模型）和一个 `.bin` 文件（QNN离线产物）。 | 是 |
| `[totalShapeNum]` | 需要支持的动态输入形状的总数量。 | 否 |
| `[inputShapeN]` | 具体的输入形状配置。根据 `totalShapeNum` 的数量，提供相应个数的形状描述。形状信息可以是以下两种格式之一：<br>1. **形状字符串**：例如 `1x3x512x512`。对于多输入模型，用下划线 `_` 分隔，例如 `1x3x512x512_1x256`。<br>2. **MNN 文件路径**：提供一个包含所需输入信息的 `.mnn` 文件路径。 | 否 |
...
[Pass]: qnn-context-binary-generator success!
[All Pass]: npu model generator success!
[Output Product]:
New mnn model path: ./qnn_models/fastvlm_visual_8_sym_57_v75.mnn
Npu model path: ./qnn_models/fastvlm_visual_8_sym_57_v75_combined.bin
#### 示例
假设 QNN SDK 路径为 /path/to/qnn/sdk，目标设备为 8Gen3 (socId=57, hexagonArch=75)，原始模型为 model.mnn，输出目录为 /path/to/output
- 使用默认输入形状进行转换
```
./MNN2QNNModel /path/to/qnn/sdk 57 75 model.mnn /path/to/output
```
### 生成多种QNN设备模型脚本
tools/script/genQNNModelsFromMNN.py中提供了8Gen1 ~ 8Elite设备的QNN模型生成脚本
- 为单输入模型指定单种输入形状进行转换
```
// 使用示例
cd mnn_path
cd build
python3 ../tools/script/genQNNModelsFromMNN.py --config_path ../source/backend/qnn/convertor/config_example/ --graph_name visual_qnn --qnn_sdk_root_path /mnt/2Tpartition/tianbu/QNN/qairt/2.37.0.250724/ --src_model visual.mnn --executable_path ./MNN2QNNModel
./MNN2QNNModel /path/to/qnn/sdk 57 75 model.mnn /path/to/output 1 1x3x256x256
```
后续将在qnn_models文件夹下生成8Gen1 ~ 8Elite设备的QNN模型产物。
- 为单输入模型指定多种输入形状进行转换
```
./MNN2QNNModel /path/to/qnn/sdk 57 75 model.mnn /path/to/output 2 1x3x256x256 1x3x512x512
```
- 为多输入模型指定多种输入形状进行转换
```
./MNN2QNNModel /path/to/qnn/sdk 57 75 model.mnn /path/to/output 2 1x3x256x256_1x100 1x3x512x512_1x200
```
#### 产物
工具执行成功后，会在指定的 `<outputDir>` 目录下生成两个文件。文件名由原始模型名、SoC ID 和 Hexagon 架构版本共同决定，格式为 `<原始模型名>_<socId>_<hexagonArch>.<suffix>`。
- **替代模型**：一个 `.mnn` 文件。文件名格式为`<原始模型名>_<socId>_<hexagonArch>.mnn`。
- **QNN离线产物**：一个 `.bin` 文件，QNN离线产物，包含了优化后的模型和权重。文件名格式为`<原始模型名>_<socId>_<hexagonArch>.bin`。
例如，对于上述示例（原始模型为 `model.mnn`，socId=57，hexagonArch=75），产物将位于 `/path/to/output/` 目录下：
```
/path/to/output/
├── model_57_75.mnn # 替代模型
└── model_57_75.bin # QNN离线产物
```
关于如何使用这些产物，可进一步参考[QNN离线构图模式的使用说明](../inference/npu.md#离线构图模式推理常规模型)。
## compilefornpu
对于较复杂的模型，通过compilefornpu及对应的`npu_convert.py`分段转换为NPU，该工具目前仅在llm相关模型的转换中使用
docs/transformers/llm.md
+164
View File
@@ -764,3 +764,167 @@ print(out)
out_ids = qwen.generate([151644, 872, 198, 108386, 151645, 198, 151644, 77091])
print(out_ids)
```
## NPU 推理 LLM
使用NPU推理，需要特定的导出参数，并针对目标设备转换出相应的模型。目前支持使用高通芯片和MTK芯片的NPU进行推理。一般流程是：LLM模型导出->转换成对应设备NPU模型->推到目标设备运行
### LLM 模型导出
NPU运行LLM需要特定的量化格式，需要按如下参数以导出 mnn
`--smooth --act_bit=16 --quant_block=0 --lm_quant_bit=16 --quant_bit=4 --seperate_embed --sym --act_sym`
eg:
```
python3 llmexport.py --path /Users/xtjiang/.cache/modelscope/hub/models/Qwen/Qwen3-4B --export mnn --smooth --act_bit=16 --quant_block=0 --lm_quant_bit=16 --seperate_embed --quant_bit=4 --sym --act_sym
```
### QNN LLM
#### 获得QNN依赖
可通过以下步骤获取依赖：
- [注册高通账号](https://myaccount.qualcomm.com/signup)
- 访问Qualcomm AI Engine Direct SDK（即QNN SDK），下载SDK，并解压。比如`/home/xiaying/third/qnn/qairt/2.38.0.250901`
- 修改`~/.bashrc` ，增加SDK路径到环境变量, 然后运行 `source ~/.bashrc` 或者重启终端。eg：
```
export QNN_SDK_ROOT=/home/xiaying/third/qnn/qairt/2.38.0.250901
export QNN_ROOT=/home/xiaying/third/qnn/qairt/2.38.0.250901
export HEXAGON_SDK_ROOT=/home/xiaying/third/qnn/qairt/2.38.0.250901
```
#### 构建 QNN 模型
在模型转换器编译时，增加`-DMNN_QNN=ON -DMNN_QNN_CONVERT_MODE=ON`，eg:
```
cd ${MNN_ROOT}
mkdir build && cd build
cmake .. -DMNN_QNN=ON -DMNN_QNN_CONVERT_MODE=ON -DMNN_BUILD_TOOLS=ON -DMNN_BUILD_LLM=ON
make -j16
```
使用 `npu/generate_llm_qnn.py` 构建 qnn 模型
eg:
```
cd ${MNN_ROOT}
cd transformers/llm/export
python3 npu/generate_llm_qnn.py --model model --soc_id=57 --dsp_arch=v75
```
目标设备`soc_id` 和 `dsp_arch` 可在高通官方查询，如下为一些设备的参考
| 硬件 | SOC ID | HEXAGON ARCH |
| :------ | :----- | :----------- |
| 8 Gen 1 | 36 | 69 |
| 8 Gen 2 | 43 | 73 |
| 8 Gen 3 | 57 | 75 |
| 8 Elite | 69 | 79 |
***执行成功后，会在 model 目录下产出 config_qnn.json 及 model/qnn 目录***
***构建完成后，model 目录下的 llm.mnn 及 llm.mnn.weight 不再需要，可以删除以减少文件总大小***
#### Android设备上运行QNN LLM
- 编译 MNN Android 库并推送到目标设备，编译时需要增加 `-DMNN_QNN=ON -DMNN_WITH_PLUGIN=ON`，eg:
```
cd ${MNN_ROOT}
cd project/android
mkdir build_64 && cd build_64
../build_64.sh -DMNN_QNN=ON -DMNN_WITH_PLUGIN=ON -DMNN_BUILD_LLM=ON -DMNN_LOW_MEMORY=ON
../updateTest.sh
```
- 参考如下脚本把 QNN 相关 so 放到 Android 对应测试目录中
```
ANDROID_WORKING_DIR=/data/local/tmp/MNN/
HEXAGON_ARCH=v75
adb push ${QNN_SDK_ROOT}/lib/aarch64-android/libQnnHtp.so ${ANDROID_WORKING_DIR}
adb push ${QNN_SDK_ROOT}/lib/aarch64-android/libQnnHtpV${HEXAGON_ARCH}Stub.so ${ANDROID_WORKING_DIR}
adb push ${QNN_SDK_ROOT}/lib/hexagon-v${HEXAGON_ARCH}/unsigned/libQnnHtpV${HEXAGON_ARCH}Skel.so ${ANDROID_WORKING_DIR}
adb push ${QNN_SDK_ROOT}/lib/aarch64-android/libQnnSystem.so ${ANDROID_WORKING_DIR}
```
- 推送模型并执行
推送模型：
```
cd ${MNN_ROOT}
cd transformers/llm/export
adb push model /data/local/tmp/MNN/model
```
运行：
```
cd ${MNN_ROOT}
project/android/testCommon.sh ./llm_demo model/config_qnn.json
```
### MTK LLM
#### 获得 MTK SDK
- 目前MTK没有开放SDK获得方案，需自行联系MTK取得支持，获得对应的SDK
- 获取后，修改`~/.bashrc`，添加环境变量，eg:
```
export NEURON_SDK=/home/xiaying/third/mtk/neuropilot-sdk-basic-7.0.8-build20240807/neuron_sdk
```
#### 构建 MLDA 模型
MLDA 是 MTK 的 NPU 推理引擎，需要把 MNN 模型转成 MLDA 模型才可在其NPU上运行
- 增加MNN对应的预转换后端配置 `-DMNN_NEUROPILOT=ON` ，eg:
```
cd ${MNN_ROOT}
mkdir build && cd build
cmake ../ -DMNN_BUILD_CONVERTER=ON -DMNN_BUILD_LLM=ON -DMNN_NEUROPILOT=ON
make -j4
```
- 确定设备的`mlda`版本号和编译选项，并修改`source/backend/neuropilot/npu_convert.py`的`archoptions`，当前默认配置为`--arch=mdla5.1 --l1-size-kb=7168 --num-mdla=4`，支持天玑9300的NPU编译
- 使用 `npu/generate_llm_mlda.py` 构建 MLDA 模型
```
cd ${MNN_ROOT}
cd transformers/llm/export
python3 npu/generate_llm_mlda.py --model model
```
执行成功后，会在 model 目录下产出`config_mlda.json`与`mlda`目录。
***生成后，原先的llm.mnn和llm.mnn.weight可以删除***
#### Android设备上运行 MLDA LLM
- 增加`-DMNN_NEUROPILOT=ON -DMNN_WITH_PLUGIN=ON`编译 MNN Android 库
```
cd ${MNN_ROOT}
cd project/android/
mkdir build_64
cd build_64
../build_64.sh -DMNN_NEUROPILOT=ON -DMNN_WITH_PLUGIN=ON -DMNN_BUILD_LLM=ON
../updateTest.sh
```
- 推送模型并执行
推送模型：
```
cd ${MNN_ROOT}
cd transformers/llm/export
adb push model /data/local/tmp/MNN/model
```
运行：
```
cd ${MNN_ROOT}
project/android/testCommon.sh ./llm_demo model/config_mlda.json
```
express/Executor.cpp
+4
View File
@@ -240,6 +240,10 @@ void Executor::RuntimeManager::setHint(Interpreter::HintMode mode, int* value, s
}
}
void Executor::RuntimeManager::setExternalPath(std::string path, int type) {
if (type == MNN::Interpreter::EXTERNAL_NPU_FILE_DIR) {
mInside->mContent->mNpuDir = path;
return;
}
mInside->mContent->modes.setExternalPath(path, type);
}
void Executor::RuntimeManager::setHintPtr(Interpreter::HintMode mode, void* value) {
express/RuntimeAttr.hpp
+1
View File
@@ -17,6 +17,7 @@ struct RuntimeAttr {
bool mUserConfig;
int mNumberThread;
std::string mExternalFile;
std::string mNpuDir;
};
std::shared_ptr<Immutable> mContent;
RuntimeInfo mRuntime;
express/Utils.cpp
+3 -2
View File
@@ -182,8 +182,9 @@ void* Executor::ComputeCache::mapOutput(int offset, Tensor* dest) {
return nullptr;
}
Utils::allocMemoryForHostTensor(dest);
tensor->copyToHostTensor(dest);
MNN_ASSERT(nullptr != dest->host<void>());
if(nullptr != dest->host<void>()) {
tensor->copyToHostTensor(dest);
}
return dest->host<void>();
}
express/module/StaticModule.cpp
+6
View File
@@ -317,6 +317,12 @@ StaticModule::StaticModule(std::vector<int> inputs,
auto& bnCache = scheduleInfo.pipelineInfo[0].first;
// Create Backend for prearrange
Session::createPipelineBackend(scheduleInfo.pipelineInfo[0], rt);
if (nullptr == bnCache.cache.first || nullptr == bnCache.cache.second) {
MNN_ERROR("[MNN:Express] Create Backend Error\n");
return;
}
bnCache.cache.first->pNPUModelDirPath = rtm->getInside()->mContent->mNpuDir;
bnCache.cache.second->pNPUModelDirPath = rtm->getInside()->mContent->mNpuDir;
if (config.rearrange) {
mResource->mBuffer = preRearrangeWeights(scheduleInfo, bnCache.cache.first.get(), bnCache.cache.second.get(), config.base);
} else {
include/MNN/MNNDefine.h
+2 -2
View File
@@ -77,7 +77,7 @@ MNN_ERROR("Check failed: %s ==> %s\n", #success, #log); \
#define STR_IMP(x) #x
#define STR(x) STR_IMP(x)
#define MNN_VERSION_MAJOR 3
#define MNN_VERSION_MINOR 2
#define MNN_VERSION_PATCH 5
#define MNN_VERSION_MINOR 3
#define MNN_VERSION_PATCH 0
#define MNN_VERSION STR(MNN_VERSION_MAJOR) "." STR(MNN_VERSION_MINOR) "." STR(MNN_VERSION_PATCH)
#endif /* MNNDefine_h */
include/MNN/MNNForwardType.h
+7 -1
View File
@@ -49,7 +49,13 @@ typedef enum {
MNN_FORWARD_CPU_EXTENSION = 13,
// use for shared memory on android device
MNN_MEMORY_AHARDWAREBUFFER = 14
MNN_MEMORY_AHARDWAREBUFFER = 14,
/* For Offline Convert*/
MNN_CONVERT_QNN = 32,
MNN_CONVERT_NEUROPILOT = 33,
MNN_CONVERT_COREML = 34,
} MNNForwardType;
typedef enum {
include/MNN/plugin/PluginContext.hpp
+9 -3
View File
@@ -24,6 +24,10 @@ public:
PluginContext() = delete;
PluginContext(const std::vector<Tensor*>& inputs, // NOLINT
const std::vector<Tensor*>& outputs);
void reset(const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs) {
inputs_ = inputs;
outputs_ = outputs;
}
virtual ~PluginContext() = default;
@@ -51,8 +55,8 @@ public:
const std::unordered_map<std::string, const Attribute*>& getAttrs() const;
protected:
const std::vector<Tensor*>& inputs_;
const std::vector<Tensor*>& outputs_;
std::vector<Tensor*> inputs_;
std::vector<Tensor*> outputs_;
std::unordered_map<std::string, const Attribute*> attrs_;
};
@@ -130,7 +134,9 @@ inline void PluginContext::setAttrs( // NOLINT
inline const Attribute* PluginContext::getAttr(const std::string& name) const {
const auto& it = attrs_.find(name);
MNN_ASSERT(it != attrs_.end());
if (it == attrs_.end()) {
return nullptr;
}
return it->second;
}
include/MNN/plugin/PluginKernel.hpp
+8
View File
@@ -35,6 +35,14 @@ public:
virtual ~CPUComputeKernel() = default;
virtual bool init(CPUKernelContext* ctx) = 0;
virtual bool compute(CPUKernelContext* ctx) = 0;
virtual bool resize(CPUKernelContext* ctx) {
return true;
}
inline bool needAllocIO() const {
return mNeedAllocIO;
}
protected:
bool mNeedAllocIO = true;
};
template <typename PluginKernelT>
schema/current/MNN_generated.h
+4 -8
View File
@@ -282,7 +282,6 @@ enum OpType {
OpType_ConvInt8 = 513,
OpType_Int8ToFloat = 514,
OpType_DepthwiseConvInt8 = 515,
OpType_PoolInt8 = 516,
OpType_FloatToInt8 = 517,
OpType_EltwiseInt8 = 518,
OpType_While = 600,
@@ -293,7 +292,7 @@ enum OpType {
OpType_MAX = OpType_GridSample
};
inline const OpType (&EnumValuesOpType())[183] {
inline const OpType (&EnumValuesOpType())[182] {
static const OpType values[] = {
OpType_AbsVal,
OpType_QuantizedAdd,
@@ -471,7 +470,6 @@ inline const OpType (&EnumValuesOpType())[183] {
OpType_ConvInt8,
OpType_Int8ToFloat,
OpType_DepthwiseConvInt8,
OpType_PoolInt8,
OpType_FloatToInt8,
OpType_EltwiseInt8,
OpType_While,
@@ -1000,7 +998,7 @@ inline const char * const *EnumNamesOpType() {
"ConvInt8",
"Int8ToFloat",
"DepthwiseConvInt8",
"PoolInt8",
"",
"FloatToInt8",
"EltwiseInt8",
"",
@@ -8074,13 +8072,12 @@ inline const flatbuffers::TypeTable *OpTypeTypeTable() {
{ flatbuffers::ET_INT, 0, 0 },
{ flatbuffers::ET_INT, 0, 0 },
{ flatbuffers::ET_INT, 0, 0 },
{ flatbuffers::ET_INT, 0, 0 },
{ flatbuffers::ET_INT, 0, 0 }
};
static const flatbuffers::TypeFunction type_refs[] = {
OpTypeTypeTable
};
static const int64_t values[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 299, 300, 301, 302, 303, 304, 512, 513, 514, 515, 516, 517, 518, 600, 601, 603, 604 };
static const int64_t values[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 299, 300, 301, 302, 303, 304, 512, 513, 514, 515, 517, 518, 600, 601, 603, 604 };
static const char * const names[] = {
"AbsVal",
"QuantizedAdd",
@@ -8258,7 +8255,6 @@ inline const flatbuffers::TypeTable *OpTypeTypeTable() {
"ConvInt8",
"Int8ToFloat",
"DepthwiseConvInt8",
"PoolInt8",
"FloatToInt8",
"EltwiseInt8",
"While",
@@ -8267,7 +8263,7 @@ inline const flatbuffers::TypeTable *OpTypeTypeTable() {
"GridSample"
};
static const flatbuffers::TypeTable tt = {
flatbuffers::ST_ENUM, 183, type_codes, type_refs, values, names
flatbuffers::ST_ENUM, 182, type_codes, type_refs, values, names
};
return &tt;
}
schema/default/MNN.fbs
-1
View File
@@ -200,7 +200,6 @@ enum OpType : int {
ConvInt8 = 513,
Int8ToFloat = 514,
DepthwiseConvInt8 = 515,
PoolInt8 = 516,
FloatToInt8 = 517,
EltwiseInt8 = 518,
source/backend/arm82/Arm82Backend.cpp
+3
View File
@@ -109,6 +109,9 @@ void Arm82Backend::onCopyBuffer(const Tensor* srcTensorC, const Tensor* dstTenso
return;
}
_resetDynamicMemory();
if (mRuntime->pCurrentStatus != NO_ERROR) {
return;
}
auto source = TensorUtils::getDescribe(srcTensor)->dimensionFormat;
auto dest = TensorUtils::getDescribe(dstTensor)->dimensionFormat;
auto srcType = _getBackendType(srcTensor);
source/backend/arm82/Arm82Functions.cpp
+198 -198
View File
File diff suppressed because it is too large Load Diff
source/backend/arm82/CMakeLists.txt
+1 -1
View File
@@ -25,7 +25,7 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR ARCHS STREQUAL "arm64" OR AR
if (MNN_LOW_MEMORY)
target_compile_options(MNN_Arm82 PRIVATE -DMNN_LOW_MEMORY)
endif()
if (MNN_CPU_WEIGHT_DEQUANT_GEMM)
target_compile_options(MNN_Arm82 PRIVATE -DMNN_CPU_WEIGHT_DEQUANT_GEMM)
endif()
source/backend/arm82/asm/arm64/sme2_asm/MNNPackedMatMulRemainFP16_SME2.S
+120 -1221
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/CPUAttention.cpp
+2 -2
View File
@@ -418,7 +418,7 @@ ErrorCode CPUAttention::onExecute(const std::vector<Tensor*>& inputs, const std:
int loop_e = seq_len / eP;
int remain = seq_len % eP;
auto qStride0 = ROUND_UP(mHeadDim, lP) * eP * bytes;
size_t shapeParameters[7] = {(size_t)eP * bytes, ROUND_UP((size_t)mHeadDim, lP), (size_t)subKvSeqLen, (size_t)seq_len * mPack * bytes, 0, 0, 0};
size_t shapeParameters[7] = {(size_t)eP * lP * bytes, ROUND_UP((size_t)mHeadDim, lP), (size_t)subKvSeqLen, (size_t)seq_len * mPack * bytes, 0, 0, 0};
for (int ei = 0 ; ei < loop_e; ei++) {
QxK((float*)(qkPacked + (ei * eP * mPack) * bytes), (float*)(qReordered + ei * qStride0), (float*)keyPtr, shapeParameters, nullptr, nullptr, (float*)scale_addr, (float*)zero_point_addr);
}
@@ -449,7 +449,7 @@ ErrorCode CPUAttention::onExecute(const std::vector<Tensor*>& inputs, const std:
}
// qk @ v
// TODO: update qkvPacked using diffScale
size_t shapeParameters[7] = {(size_t)eP * bytes, ROUND_UP((size_t)subKvSeqLen, lP), (size_t)mHeadDim, (size_t)seq_len * mPack * bytes, 0, 0, 0};
size_t shapeParameters[7] = {(size_t)eP * lP * bytes, ROUND_UP((size_t)subKvSeqLen, lP), (size_t)mHeadDim, (size_t)seq_len * mPack * bytes, 0, 0, 0};
size_t bExtraStride = (UP_DIV(max_len, lP) - UP_DIV(subKvSeqLen + i * mBlockKV, lP) + UP_DIV(i * mBlockKV, lP)) * hP * lP * bytes;
shapeParameters[5] = quant_value ? 0 : bExtraStride;
int loop_e = seq_len / eP;
source/backend/cpu/CPUBackend.cpp
+13 -8
View File
@@ -326,7 +326,7 @@ Backend* CPURuntime::onCreate(const BackendConfig* config, Backend* origin) cons
auto core = MNNGetCoreFunctions();
if (core->supportFp16arith && precision == BackendConfig::Precision_Low) {
res = new Arm82Backend(this, memory);
if (hint().useArmSme2Cores && res->threadNumber() <= 2 && core->supportSME2 && res->functions()->sme2Int8MatmulRelatedFuncionsHp32.Int8GemmKernel) {
if (hint().useArmSme2Cores && core->supportSME2 && res->functions()->sme2Int8MatmulRelatedFuncionsHp32.Int8GemmKernel) {
res->mRelatedFunctions = &(res->functions()->sme2Int8MatmulRelatedFuncionsHp32);
} else {
res->mRelatedFunctions = &(res->functions()->int8MatmulRelatedFunctions);
@@ -656,12 +656,6 @@ static OpType _getRealOpType(OpType opType) {
return OpType_ConvInt8;
case OpType_ConvolutionDepthwise:
return OpType_DepthwiseConvInt8;
case OpType_Pooling:
return OpType_PoolInt8;
// case OpType_Eltwise:
// // TODO: just support EltwiseAdd
// return OpType_EltwiseInt8;
default:
return opType;
}
@@ -674,6 +668,10 @@ void* CPUBackend::onMapTensor(Tensor::MapType mtype, Tensor::DimensionType dtype
return nullptr;
}
_resetDynamicMemory();
if (mRuntime->pCurrentStatus != NO_ERROR) {
// Out of memory
return nullptr;
}
return srcTensor->host<void>();
}
@@ -785,6 +783,10 @@ std::pair<int, int> CPUBackend::multiThreadDivide(int size) const {
}
void CPUBackend::onCopyBuffer(const Tensor* srcTensor, const Tensor* dstTensor) const {
_resetDynamicMemory();
if (mRuntime->pCurrentStatus != NO_ERROR) {
// Out of memory
return;
}
auto& srcBuffer = srcTensor->buffer();
auto& dstBuffer = dstTensor->buffer();
if (srcBuffer.dimensions != dstBuffer.dimensions ) {
@@ -874,7 +876,7 @@ public:
case OpType_Raster:
{
for (auto input : inputs) {
if (TensorUtils::getDescribe(input)->quantAttr.get() != TensorUtils::getDescribe(outputs[0])->quantAttr.get()) {
if (TensorUtils::getDescribe(input)->quantAttr->scale != TensorUtils::getDescribe(outputs[0])->quantAttr->scale || TensorUtils::getDescribe(input)->quantAttr->zero != TensorUtils::getDescribe(outputs[0])->quantAttr->zero) {
return false;
}
if (TensorUtils::getDescribe(input)->quantAttr.get() && TensorUtils::getDescribe(outputs[0])->quantAttr.get() && (TensorUtils::getDescribe(input)->quantAttr.get()->scale == 0 || TensorUtils::getDescribe(outputs[0])->quantAttr.get()->scale == 0)) {
@@ -884,6 +886,9 @@ public:
return true;
}
case OpType_Pooling:
if (TensorUtils::getDescribe(inputs[0])->quantAttr->scale != TensorUtils::getDescribe(outputs[0])->quantAttr->scale || TensorUtils::getDescribe(inputs[0])->quantAttr->zero != TensorUtils::getDescribe(outputs[0])->quantAttr->zero) {
return false;
}
if (op->main_as_Pool() && op->main_as_Pool()->type() == PoolType_MAXPOOL ) {
return true;
} else if (op->main_as_Pool() && op->main_as_Pool()->type() == PoolType_AVEPOOL) {
source/backend/cpu/CPUBackend.hpp
+1 -1
View File
@@ -188,6 +188,7 @@ protected:
CoreFunctions* mCoreFunctions;
CoreInt8Functions* mInt8CoreFunctions;
const MatmulRelatedFunctions* mRelatedFunctions;
CPURuntime* mRuntime;
private:
mutable std::shared_ptr<WorkerThread> mInitWorkQueue;
mutable int mThreadNumber = 1;
@@ -195,7 +196,6 @@ private:
float mComputeI = 0.f;
std::shared_ptr<CPURuntime::DynamicAllocator> mDmaInfo;
CPURuntime* mRuntime;
BackendConfig::PrecisionMode mPrecisionMode;
BackendConfig::MemoryMode mMemory;
static std::map<OpType, CPUBackend::Creator*>* gCreator;
source/backend/cpu/CPUConvolution.cpp
+17 -16
View File
@@ -32,7 +32,7 @@ static void unpackScaleFromBuffer(float* scaleBuffer, const int8_t* srcbuffer, c
int stride1 = info[2]; // int stride1 = blockL * UNIT * SRC_UNIT; // Int8 weight size per block
int UNIT = info[3];
size_t packedUnitSize = stride1 + 2 * UNIT * infoBytes;
int8_t* scaleWritePtr = reinterpret_cast<int8_t*>(scaleBuffer);
@@ -80,8 +80,9 @@ bool CPUConvolution::Resource::copyBiasAlign(const float* bias, int outputCount)
return true;
}
CPUConvolution::MutableResourceInt8::MutableResourceInt8(std::shared_ptr<ResourceInt8> res, Backend* backend, float* scalePtr) : mResource(res) {
auto outputChannleUp4 = res->mOriginBias->length(0); // outputChannleUp4 = ROUND_UP(oc, UNIT)
mBiasFloat.reset(Tensor::createDevice<int32_t>({outputChannleUp4}));
auto outputChannelUp4 = res->mOriginBias->length(0); // outputChannelUp4 = ROUND_UP(oc, pack)
const int ocUpHp = (int)(res->mWeightKernelSum->length(0) / res->mBlockNum / sizeof(float));
mBiasFloat.reset(Tensor::createDevice<int32_t>({outputChannelUp4}));
mValid = backend->onAcquireBuffer(mBiasFloat.get(), Backend::STATIC);
if (!mValid) {
MNN_ERROR("mBiasFloat buffer allocated error!\n");
@@ -101,9 +102,9 @@ CPUConvolution::MutableResourceInt8::MutableResourceInt8(std::shared_ptr<Resourc
auto int32BiasPtr = res->mOriginBias->host<int32_t>();
auto floatBiasPtr = mBiasFloat->host<float>();
auto weightScale = scalePtr;
auto blockNum = res->mBlockNum;
AutoStorage<int8_t> tmpBuffer(outputChannleUp4 * blockNum * 4);
AutoStorage<int8_t> tmpBuffer(ocUpHp * blockNum * 4);
if (!tmpBuffer.get()) {
MNN_ERROR("Memory not enough for allocating a temp buffer for weight scale\n");
return;
@@ -113,15 +114,15 @@ CPUConvolution::MutableResourceInt8::MutableResourceInt8(std::shared_ptr<Resourc
int UNIT, SRC_UNIT, DST_XUNIT;
auto int8Core = static_cast<CPUBackend*>(backend)->int8Functions();
int8Core->MNNGetGemmUnit(&UNIT, &SRC_UNIT, &DST_XUNIT);
int32_t perBlockWeightSize = (res->mWeightInt8->size() - 2 * outputChannleUp4 * sizeof(float)) / (blockNum * UP_DIV(outputChannleUp4, UNIT));
int32_t info[4] = {blockNum, UP_DIV(outputChannleUp4, UNIT), perBlockWeightSize, UNIT};
int32_t perBlockWeightSize = (res->mWeightInt8->size() - 2 * ocUpHp * sizeof(float)) / (blockNum * UP_DIV(ocUpHp, UNIT));
int32_t info[4] = {blockNum, UP_DIV(ocUpHp, UNIT), perBlockWeightSize, UNIT};
unpackScaleFromBuffer((float*)tmpBuffer.get(), res->mWeightInt8->host<int8_t>(), info, 4);
weightScale = (float*)tmpBuffer.get();
} else if (!scalePtr) { // if depthwiseInt8, res->mOriginScale != nullptr
weightScale = res->mOriginScale->host<float>();
}
for (int i = 0; i < outputChannleUp4; ++i) {
for (int i = 0; i < outputChannelUp4; ++i) {
if (mInputScale && mOutputScale) { // symmetric quan
floatBiasPtr[i] = int32BiasPtr[i] * weightScale[i] * mInputScale / mOutputScale;
} else {
@@ -130,8 +131,8 @@ CPUConvolution::MutableResourceInt8::MutableResourceInt8(std::shared_ptr<Resourc
}
return;
}
mBiasInt32.reset(Tensor::createDevice<int32_t>({outputChannleUp4}));
mScaleFloat.reset(Tensor::createDevice<int32_t>({outputChannleUp4}));
mBiasInt32.reset(Tensor::createDevice<int32_t>({outputChannelUp4}));
mScaleFloat.reset(Tensor::createDevice<int32_t>({outputChannelUp4}));
mValid = backend->onAcquireBuffer(mBiasInt32.get(), Backend::STATIC);
if (mValid) {
mValid = backend->onAcquireBuffer(mScaleFloat.get(), Backend::STATIC);
@@ -165,7 +166,7 @@ void CPUConvolution::MutableResourceInt8::updateInputOutputScale(std::vector<flo
return;
}
const int ocUp4 = static_cast<int>(mResource->mWeightKernelSum->length(0) / sizeof(float));
const int ocUp4 = mResource->mOriginBias->length(0);
auto biasData = mResource->mOriginBias->host<float>();
auto scaleDiv = mInputScale / mOutputScale;
auto scale = mScaleFloat->host<float>();
@@ -191,7 +192,7 @@ void CPUConvolution::MutableResourceInt8::updateInputOutputScale(std::vector<flo
} else {
for (int i = 0; i < ocUp4; ++i) {
biasfloat[i] = (biasData[i] - mResource->mWeightKernelSum->host<float>()[i] * (mInputZeroPoint + offset) * mInputScale) / mOutputScale + mOutputZeroPoint;
}
}
}
@@ -321,7 +322,7 @@ public:
return OneDNNConvInt8::create(backend, op, inputs, outputs);
#endif
auto core = static_cast<CPUBackend*>(backend)->functions();
#ifdef MNN_USE_SPARSE_COMPUTE
if (static_cast<CPUBackend*>(backend)->functions()->pack == 4 && convOp->sparseParameter() && SparseConvInt8TiledExecutor::shouldUseSparse(convOp)) {
auto res = CPUConvolution::makeResourceInt8(backend, op, core->pack);
@@ -337,11 +338,11 @@ public:
std::shared_ptr<ConvolutionCommon::Int8Common> quanCommon;
if (convOp->quanParameter() && (convOp->quanParameter()->buffer() || convOp->external())) { // int8 weight
quanCommon = ConvolutionCommon::load(op, backend, false, true);
}
// auto res = CPUConvolution::makeResourceInt8(backend, op, core->pack);
// return new DenseConvInt8TiledExecutor(backend, op, res);
return new DenseConvInt8TiledExecutor(backend, op, quanCommon, false);
}
};
source/backend/cpu/CPUDeconvolution.cpp
+20 -11
View File
@@ -97,6 +97,9 @@ std::shared_ptr<DeconvolutionResource> CPUDeconvolution::makeResource(int srcCou
MNN_ERROR("Alloc memory error for deconvolution\n");
return nullptr;
}
if (lP > 1) {
memset(res->mWeight->host<uint8_t>(), 0, res->mWeight->length(0) * res->mWeight->stride(0) * core->bytes);
}
CPUConvolution::Resource::copyBias(res->mBias->host<float>(), convOp->main_as_Convolution2D()->bias()->data(), outputCount, backend);
_transformWeight((uint8_t*)tempWeight, res->mWeight->host<uint8_t>(), outputCount, srcCount, fh, fw, (uint8_t*)cache.get(), core);
return res;
@@ -147,6 +150,12 @@ CPUDeconvolution::~CPUDeconvolution() {
ErrorCode CPUDeconvolution::onExecute(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs) {
if (mDynamicWeight) {
auto core = static_cast<CPUBackend*>(backend())->functions();
int eP, lP, hP;
core->MNNGetMatMulPackMode(&eP, &lP, &hP);
if (lP > 1) {
memset(mWeight->host<uint8_t>(), 0, mWeight->length(0) * mWeight->stride(0) * core->bytes);
}
_transformWeight(inputs[1]->host<uint8_t>(), mWeight->host<uint8_t>(), mResource->mParam.outputCount, mResource->mParam.srcCount, mResource->mParam.fh, mResource->mParam.fw, mWeightTransformCache->host<uint8_t>(), core);
::memset(mBias->host<uint8_t>(), 0, mBias->length(0) * core->bytes);
if (inputs.size() >= 3) {
@@ -229,7 +238,7 @@ ErrorCode CPUDeconvolutionOrigin::onResize(const std::vector<Tensor*>& inputs, c
// Limit threadNumber to avoid too large memory
threadNumber = ALIMIN(threadNumber, 4);
}
auto im2colOutputStride = input->channel() * eP * core->bytes;
auto im2colOutputStride = ROUND_UP(input->channel(), lP) * eP * core->bytes;
mGemmInput = allocator->alloc(threadNumber * im2colOutputStride);
auto gemmOutputStride = kernelCount * core->pack * eP * core->bytes;
mGemmOutput = allocator->alloc(threadNumber * gemmOutputStride);
@@ -264,7 +273,7 @@ ErrorCode CPUDeconvolutionOrigin::onResize(const std::vector<Tensor*>& inputs, c
continue;
}
size_t parameters[7];
parameters[0] = xCount * core->bytes;
parameters[0] = xCount * lP * core->bytes;
parameters[1] = l;
parameters[2] = h;
parameters[3] = xCount * core->bytes * core->pack;
@@ -294,26 +303,26 @@ ErrorCode CPUDeconvolutionOrigin::onResize(const std::vector<Tensor*>& inputs, c
auto dstZ = tempOutPtr + z * src_height * src_width * batch * unitBytes;
auto srcZ = colBufferPtr + kw * kh * xCount * z * unitBytes;
for (int x=0; x<xCount; ++x) {
auto index = xStart + x;
int b = index / (width * height);
index = index % (width * height);
int oy = index / width;
int ox = index % width;
auto indexE = xStart + x;
int b = indexE / (width * height);
indexE = indexE % (width * height);
int oy = indexE / width;
int ox = indexE % width;
int srcStartX = ox * strideX - padX;
int srcStartY = oy * strideY - padY;
int sfy = ALIMAX(0, (UP_DIV(-srcStartY, dilateY)));
int efy = ALIMIN(kh, UP_DIV(src_height - srcStartY, dilateY));
int sfx = ALIMAX(0, (UP_DIV(-srcStartX, dilateX)));
int efx = ALIMIN(kw, UP_DIV(src_width - srcStartX, dilateX));
auto dstStart = dstZ + b * src_width * src_height * unitBytes + srcStartX * unitBytes + srcStartY * src_width * unitBytes;
auto srcStart = srcZ + x * unitBytes;
if (sfy >= efy || sfx >= efx) {
continue;
}
for (int fy = sfy; fy < efy; ++fy) {
auto dstY = dstStart + fy * unitBytes * dilateY * src_width;
auto srcY = srcStart + fy * kw * xCount * unitBytes;
source/backend/cpu/CPUMatMul.cpp
+2 -1
View File
@@ -193,7 +193,7 @@ void CPUMatMul::execute(const float* APtr, const float* BPtr, float* CPtr, const
auto hC4 = UP_DIV(mH, core->pack);
auto TC = mTempC.ptr() + tId * eP * hC4 * core->pack * core->bytes;
size_t parameters[6];
parameters[0] = eP * core->bytes;
parameters[0] = eP * lP * core->bytes;
parameters[1] = lAlign;
parameters[2] = mH;
parameters[3] = eP * core->pack * core->bytes;
@@ -281,6 +281,7 @@ void CPUMatMul::execute(const float* APtr, const float* BPtr, float* CPtr, const
auto dst = (uint8_t*)CPtr + xStart * core->bytes;
core->MNNUnpackCUnit((float*)dst, (const float*)TC, xC, mH, area);
}
}
};
MNN_CONCURRENCY_END();
source/backend/cpu/CPUOPRegister.cpp
-2
View File
@@ -43,7 +43,6 @@ extern void ___CPUUniqueCreator__OpType_Unique__();
extern void ___CPUImageProcessCreator__OpType_ImageProcess__();
extern void ___CPUDepthwiseConvInt8Creator__OpType_DepthwiseConvInt8__();
extern void ___CPUOneHotCreator__OpType_OneHot__();
extern void ___CPUPoolInt8Creator__OpType_PoolInt8__();
extern void ___CPUMatrixBandPartCreator__OpType_MatrixBandPart__();
extern void ___CPUQuantizedAddCreator__OpType_QuantizedAdd__();
extern void ___CPUDeconvolutionDepthwiseCreator__OpType_DeconvolutionDepthwise__();
@@ -123,7 +122,6 @@ ___CPUUniqueCreator__OpType_Unique__();
___CPUImageProcessCreator__OpType_ImageProcess__();
___CPUDepthwiseConvInt8Creator__OpType_DepthwiseConvInt8__();
___CPUOneHotCreator__OpType_OneHot__();
___CPUPoolInt8Creator__OpType_PoolInt8__();
___CPUMatrixBandPartCreator__OpType_MatrixBandPart__();
___CPUQuantizedAddCreator__OpType_QuantizedAdd__();
___CPUDeconvolutionDepthwiseCreator__OpType_DeconvolutionDepthwise__();
source/backend/cpu/CPUPlugin.cpp
+16 -7
View File
@@ -32,9 +32,13 @@ public:
MNN_CHECK(nullptr != kernel_.get(), // NOLINT
"CPU compute kernel has not been registered for plugin op.");
kernel_->init(ctx_.get());
mNeedAllocIO = kernel_->needAllocIO();
}
virtual ~CPUPlugin() = default;
virtual ErrorCode onResize(const std::vector<Tensor*>& inputs, // NOLINT
const std::vector<Tensor*>& outputs) override;
virtual ErrorCode onExecute(const std::vector<Tensor*>& inputs, // NOLINT
const std::vector<Tensor*>& outputs) override;
@@ -42,17 +46,22 @@ private:
std::unique_ptr<plugin::CPUKernelContext> ctx_;
std::shared_ptr<plugin::CPUComputeKernel> kernel_;
};
ErrorCode CPUPlugin::onResize(const std::vector<Tensor*>& inputs,
const std::vector<Tensor*>& outputs) {
ctx_->reset(inputs, outputs);
auto success = kernel_->resize(ctx_.get());
if (!success) {
return OUT_OF_MEMORY;
}
return NO_ERROR;
}
ErrorCode CPUPlugin::onExecute(const std::vector<Tensor*>& inputs, // NOLINT
const std::vector<Tensor*>& outputs) {
// Setup new context with inputs and outputs.
plugin::CPUKernelContext ctx( // NOLINT
ctx_->op_type(), ctx_->backend(), inputs, outputs, ctx_->dir_path());
ctx.setAttrs(ctx_->getAttrs());
if (kernel_->compute(&ctx)) {
if (kernel_->compute(ctx_.get())) {
return NO_ERROR;
} else {
MNN_ERROR("Plugin kernel compute failed with false returned.");
MNN_ERROR("Plugin kernel compute failed with false returned.\n");
return INVALID_VALUE;
}
}
@@ -73,7 +82,7 @@ public:
const std::string& op_type = plugin_param->type()->str();
std::unique_ptr<plugin::CPUKernelContext> ctx( // NOLINT
new plugin::CPUKernelContext(op_type, backend, inputs, outputs, static_cast<CPUBackend *>(backend)->getRuntime()->hint().npuModelDirPath));
new plugin::CPUKernelContext(op_type, backend, inputs, outputs, static_cast<CPUBackend *>(backend)->pNPUModelDirPath));
for (const Attribute* attr : *(plugin_param->attr())) {
ctx->setAttr(attr->key()->str(), attr);
source/backend/cpu/CPUPool.cpp
+8
View File
@@ -8,6 +8,7 @@
#include "backend/cpu/CPUBackend.hpp"
#include "core/Concurrency.h"
#include "backend/cpu/CPUPool.hpp"
#include "backend/cpu/CPUPoolInt8.hpp"
#include "compute/CommonOptFunction.h"
#include "math/Vec.hpp"
#include "core/TensorUtils.hpp"
@@ -117,6 +118,13 @@ public:
const MNN::Op *op, Backend *backend) const override {
void* func = nullptr;
bool returnRedice = false;
bool useInt8Pool = false;
useInt8Pool = CPUBackend::getDataType(inputs[0]) == DataType_DT_INT8 || inputs[0]->getType().bytes() == 1;
useInt8Pool &= (CPUBackend::getDataType(outputs[0]) == DataType_DT_INT8 || outputs[0]->getType().bytes() == 1);
if (useInt8Pool) {
return new CPUPoolInt8(backend, op->main_as_Pool());
}
if (inputs[0]->getType() == halide_type_of<int8_t>()) {
if (op->main_as_Pool()->type() == PoolType_AVEPOOL) {
func = (void*)(poolingAvg<int8_t, Vec16, 4>);
source/backend/cpu/CPUPoolInt8.cpp
+6 -15
View File
@@ -67,7 +67,7 @@ static void poolingAvgNC16HW16Int8(void poolfunc(int8_t*, int8_t*, size_t, size_
ix = std::max(ix, 0);
int mul = static_cast<int>((1 << 24)/(kernelx_ * kernely_));
const int indexOutput = pack * (ox + outputWidth * (oy + outputHeight * (ob + batchsize * oc)));
const int indexOutput = pack * (ox + outputWidth * (oy + outputHeight * (ob + batchsize * oc)));
const int indexInput = pack * (ix + inputWidth * (iy + inputHeight * (ob + batchsize * oc)));
int8_t* dstCur = dstPtr + indexOutput;
@@ -90,7 +90,7 @@ static void poolingAvgNC16HW16Int8(void poolfunc(int8_t*, int8_t*, size_t, size_
poolfunc(dstCur, srcCur, 1, inputWidth, kernelx_, kernely_, stridesx, paddingx, mul);
}
}
}
}
}
@@ -116,7 +116,7 @@ static void poolingMaxNC16HW16Int8(void poolfunc(int8_t*, int8_t*, size_t, size_
for (int oc = 0; oc < channel16; ++oc){
for(int ob = 0; ob < batchsize; ++ob){
for (int oy = 0; oy < outputHeight; ++oy) {
int iy = oy * stridesy - paddingy;
const int kernely_ = std::min(iy + kernely, inputHeight) - std::max(iy, 0);
iy = std::max(iy, 0);
@@ -141,7 +141,7 @@ static void poolingMaxNC16HW16Int8(void poolfunc(int8_t*, int8_t*, size_t, size_
const int kernelx_ = std::min(ix + kernelx, inputWidth) - std::max(ix, 0);
ix = std::max(ix, 0);
const int indexOutput = pack * (ox + outputWidth * (oy + outputHeight * (ob + batchsize * oc)));
const int indexOutput = pack * (ox + outputWidth * (oy + outputHeight * (ob + batchsize * oc)));
const int indexInput = pack * (ix + inputWidth * (iy + inputHeight * (ob + batchsize * oc)));
int8_t* dstCur = dstPtr + indexOutput;
@@ -207,7 +207,7 @@ ErrorCode CPUPoolInt8::onResize(const std::vector<Tensor *> &inputs, const std::
}
const int channel = input->channel();
mThreadFunction = [=](const Tensor *src, Tensor *dst) {
poolingMaxNC16HW16Int8(core->MNNMaxPoolInt8, src, dst, strideWidth, strideHeight, kernelWidth, kernelHeight, padWidth, padHeight);
};
@@ -239,7 +239,7 @@ ErrorCode CPUPoolInt8::onExecute(const std::vector<Tensor *> &inputs, const std:
auto plane_out = output->width() * output->height() * output->batch();
auto core = static_cast<CPUBackend*>(backend())->functions();
auto depth = UP_DIV(channel_input, core->pack);
if (core->pack == 8) {
MNNPackC2Origin(mInputTemp.get()->host<double>(), input->host<double>(), plane_in, depth, plane_in);
mThreadFunction(mInputTemp.get(), mOutputTemp.get());
@@ -256,14 +256,5 @@ ErrorCode CPUPoolInt8::onExecute(const std::vector<Tensor *> &inputs, const std:
return NO_ERROR;
}
class CPUPoolInt8Creator : public CPUBackend::Creator {
public:
virtual Execution *onCreate(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs,
const MNN::Op *op, Backend *backend) const override {
return new CPUPoolInt8(backend, op->main_as_Pool());
}
};
REGISTER_CPU_OP_CREATOR(CPUPoolInt8Creator, OpType_PoolInt8);
} // namespace MNN
source/backend/cpu/CPURaster.cpp
+1 -1
View File
@@ -687,7 +687,7 @@ ErrorCode CPURaster::onExecute(const std::vector<Tensor *> &____inputs, const st
for (auto& iter : mTempInput) {
tensorConvert(iter.first, iter.second, (int)bytes);
}
if (mHasReduce) {
if (mHasReduce || TensorUtils::getDescribe(output)->overlap) {
// Don't support reduce with multi thread now
threadNum = 1;
}
source/backend/cpu/arm/CMakeLists.txt
+1 -1
View File
@@ -32,7 +32,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^armv7" OR ARCHS MATCHES "^armv7(;armv7s)?")
endif()
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR ARCHS STREQUAL "arm64" OR ARCHS STREQUAL "ARM64")
message(STATUS "Enabling AArch64 Assemblies")
if (MNN_KLEIDIAI)
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/KleidiAI.cmake)
download_kleidiai_and_collect_sources()
source/backend/cpu/arm/arm64/MNNGemmInt8AddBiasScale_16x4_Unit.S
+15 -18
View File
@@ -71,7 +71,7 @@
asm_function MNNGemmInt8AddBiasScale_16x4_Unit
/*
/*
struct QuanPostTreatParameters {
const float* scale;
const float* biasFloat;
@@ -92,7 +92,7 @@ struct QuanPostTreatParameters {
//void MNNGemmInt8AddBiasScale_16x4_Unit(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step,
// size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realSize) {
//Auto: x0: dst*, x1: src*, x2:weight*, x3: src_depth_quad, x4: dst_step,
//Auto: x0: dst*, x1: src*, x2:weight*, x3: src_depth_quad, x4: dst_step,
// x5: dst_depth_quad, x6: post, x7: realSize
//Load from post:
@@ -144,10 +144,10 @@ L4LoopDz:
mov x8, x1
mov x25, #0
L4_BLOCKNUM:
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64
ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #64
smull v8.8h, v0.8b, v4.8b
smull v9.8h, v1.8b, v4.8b
smull v10.8h, v2.8b, v4.8b
@@ -204,7 +204,7 @@ L4LoopDz:
saddlp v31.4s, v15.8h
L4InitializeEnd:
beq ComputeSum
L4LoopSz:
ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #64
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64
@@ -304,8 +304,6 @@ L4LoopDz:
MLA_WEIGHTZERO v6, v20, v21, 2
MLA_WEIGHTZERO v7, v20, v21, 3
cbz x28, L4QuantUseInt8
cbz x27, L4_ADD_DSTV
ld1 {v20.4s}, [x27], #16 // input dequant bias
ld1 {v21.4s}, [x28], #16 // weight kernel sum
@@ -330,6 +328,7 @@ L4LoopDz:
b L4_BLOCKNUM
L4_POST:
cbz x28, L4QuantUseInt8
cbz x10, L4_RELU
ld1 {v0.4s}, [x10], #16
fadd v4.4s, v4.4s, v0.4s
@@ -404,7 +403,7 @@ L3LoopDz:
smull v13.8h, v1.8b, v5.8b
smull v14.8h, v2.8b, v5.8b
smull v15.8h, v3.8b, v5.8b
smlal2 v8.8h, v0.16b, v4.16b
smlal2 v9.8h, v1.16b, v4.16b
smlal2 v10.8h, v2.16b, v4.16b
@@ -528,8 +527,6 @@ L3LoopDz:
MLA_WEIGHTZERO v5, v20, v21, 1
MLA_WEIGHTZERO v6, v20, v21, 2
cbz x28, L3QuantUseInt8
cbz x27, L3_ADD_DSTV
ld1 {v20.2s}, [x27], #8 // input dequant bias
ld1 {v20.s}[2], [x27], #4
@@ -553,6 +550,7 @@ L3LoopDz:
b L3_BLOCKNUM
L3_POST:
cbz x28, L3QuantUseInt8
cbz x10, L3_RELU
ld1 {v0.4s}, [x10], #16
fadd v4.4s, v4.4s, v0.4s
@@ -591,7 +589,7 @@ L3LoopDz:
smax v2.8b, v31.8b, v2.8b
smax v3.8b, v31.8b, v3.8b
st1 {v2.8b}, [x0], #8
st1 {v3.s}[0], [x0], x4
L3LoopCheck:
@@ -615,8 +613,8 @@ L2LoopDz:
L2_BLOCKNUM:
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64
ld1 {v4.16b, v5.16b}, [x1], x24
smull v8.8h, v0.8b, v4.8b
smull v9.8h, v1.8b, v4.8b
smull v10.8h, v2.8b, v4.8b
@@ -711,7 +709,6 @@ L2LoopDz:
MLA_WEIGHTZERO v4, v20, v21, 0
MLA_WEIGHTZERO v5, v20, v21, 1
cbz x28, L2QuantUseInt8
cbz x27, L2_ADD_DSTV
ld1 {v20.2s}, [x27], #8 // input dequant bias
@@ -733,6 +730,7 @@ L2LoopDz:
b L2_BLOCKNUM
L2_POST:
cbz x28, L2QuantUseInt8
cbz x10, L2_RELU
ld1 {v0.4s}, [x10], #16
fadd v4.4s, v4.4s, v0.4s
@@ -764,7 +762,7 @@ L2LoopDz:
smin v8.8b, v30.8b, v8.8b
smax v9.8b, v31.8b, v9.8b
st1 {v2.8b}, [x0], x4
L2LoopCheck:
subs x5, x5, #1
@@ -788,7 +786,7 @@ L1LoopDz:
dup v16.4s, wzr
dup v17.4s, wzr
ld1 {v4.16b}, [x1], #16
smull v8.8h, v0.8b, v4.8b
dup v18.4s, wzr
smull v9.8h, v1.8b, v4.8b
@@ -854,8 +852,6 @@ L1LoopDz:
TILE1_MUL_OHE_SCALE:
MLA_WEIGHTZERO v4, v20, v21, 0
cbz x28, L1QuantUseInt8
cbz x27, L1_ADD_DSTV
ld1 {v20.s}[0], [x27], #4 // input dequant bias
ld1 {v21.4s}, [x28], #16 // weight kernel sum
@@ -874,6 +870,7 @@ L1LoopDz:
b L1_BLOCKNUM
L1_POST:
cbz x28, L1QuantUseInt8
cbz x10, L1_RELU
ld1 {v0.4s}, [x10], #16
fadd v4.4s, v4.4s, v0.4s
source/backend/cpu/arm/arm64/MNNGemmInt8AddBiasScale_ARMV82_Unit.S
+22 -19
View File
@@ -98,7 +98,7 @@ mul \rg1, \rg2, \rg1
sub \rg0, \rg0, \rg1, LSL #5 // revert weight kernel sum
.endm
asm_function MNNGemmInt8AddBiasScale_ARMV82_Unit
/*
/*
struct QuanPostTreatParameters {
const float* scale;
const float* biasFloat;
@@ -118,7 +118,7 @@ struct QuanPostTreatParameters {
};
*/
//void MNNGemmInt8AddBiasScale_ARMV82_Unit(int8_t* dst, const int8_t* src,
//void MNNGemmInt8AddBiasScale_ARMV82_Unit(int8_t* dst, const int8_t* src,
// const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad,
// const QuanPostTreatParameters* parameters, size_t realDstCount);
@@ -251,7 +251,7 @@ TILE12_BLOCKNUM:
MLA_WEIGHTZERO v15, v3, v5, 3 // tile:7, oc:0-3
MLA_WEIGHTZERO v16, v4, v5, 0 // tile:8, oc:0-3
MLA_WEIGHTZERO v17, v4, v5, 1 // tile:9, oc:0-3
MLA_WEIGHTZERO v18, v4, v5, 2 // tile:10, oc:0-3
MLA_WEIGHTZERO v18, v4, v5, 2 // tile:10, oc:0-3
MLA_WEIGHTZERO v19, v4, v5, 3 // tile:11, oc:0-3
MLA_WEIGHTZERO v20, v2, v6, 0 // tile:0, oc:4-7
@@ -260,14 +260,13 @@ TILE12_BLOCKNUM:
MLA_WEIGHTZERO v23, v2, v6, 3 // tile:3, oc:4-7
MLA_WEIGHTZERO v24, v3, v6, 0 // tile:4, oc:4-7
MLA_WEIGHTZERO v25, v3, v6, 1 // tile:5, oc:4-7
MLA_WEIGHTZERO v26, v3, v6, 2 // tile:6, oc:4-7
MLA_WEIGHTZERO v26, v3, v6, 2 // tile:6, oc:4-7
MLA_WEIGHTZERO v27, v3, v6, 3 // tile:7, oc:4-7
MLA_WEIGHTZERO v28, v4, v6, 0 // tile:8, oc:4-7
MLA_WEIGHTZERO v29, v4, v6, 1 // tile:9, oc:4-7
MLA_WEIGHTZERO v30, v4, v6, 2 // tile:10, oc:4-7
MLA_WEIGHTZERO v31, v4, v6, 3 // tile:11, oc:4-7
cbz x28, L8Tile12QuanUseInt8
cbz x27, TILE12_ADD_DSTV
ld1 {v0.4s, v1.4s, v2.4s}, [x27], #48 // input dequant bias
@@ -291,7 +290,7 @@ TILE12_BLOCKNUM:
MLA_WEIGHTZERO v23, v0, v4, 3 // tile:3, oc:4-7
MLA_WEIGHTZERO v24, v1, v4, 0 // tile:4, oc:4-7
MLA_WEIGHTZERO v25, v1, v4, 1 // tile:5, oc:4-7
MLA_WEIGHTZERO v26, v1, v4, 2 // tile:6, oc:4-7
MLA_WEIGHTZERO v26, v1, v4, 2 // tile:6, oc:4-7
MLA_WEIGHTZERO v27, v1, v4, 3 // tile:7, oc:4-7
MLA_WEIGHTZERO v28, v2, v4, 0 // tile:8, oc:4-7
MLA_WEIGHTZERO v29, v2, v4, 1 // tile:9, oc:4-7
@@ -326,6 +325,7 @@ TILE12_BLOCKNUM:
b TILE12_BLOCKNUM
TILE12_POST:
cbz x28, L8Tile12QuanUseInt8
sub x5, x5, #2
cbz x9, TILE12_RELU
ld1 {v0.4s, v1.4s}, [x20], #32
@@ -469,11 +469,9 @@ L4_TILE12_BLOCKNUM:
MLA_WEIGHTZERO v15, v3, v5, 3 // tile:7, oc:0-3
MLA_WEIGHTZERO v16, v4, v5, 0 // tile:8, oc:0-3
MLA_WEIGHTZERO v17, v4, v5, 1 // tile:9, oc:0-3
MLA_WEIGHTZERO v18, v4, v5, 2 // tile:10, oc:0-3
MLA_WEIGHTZERO v18, v4, v5, 2 // tile:10, oc:0-3
MLA_WEIGHTZERO v19, v4, v5, 3 // tile:11, oc:0-3
cbz x28, L4Tile12QuanUseInt8
cbz x27, L4_TILE12_ADD_DSTV
ld1 {v0.4s, v1.4s, v2.4s}, [x27], #48 // input dequant bias
ld1 {v3.4s}, [x28] // weight kernel sum
@@ -512,6 +510,7 @@ L4_TILE12_BLOCKNUM:
b L4_TILE12_BLOCKNUM
L4_TILE12_POST:
cbz x28, L4Tile12QuanUseInt8
cbz x9, L4_TILE12_RELU
ld1 {v0.4s}, [x20], #16
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
@@ -651,7 +650,6 @@ TILE8_BLOCKNUM:
MLA_WEIGHTZERO v21, v3, v25, 1 // tile:5, oc:4-7
MLA_WEIGHTZERO v22, v3, v25, 2 // tile:6, oc:4-7
MLA_WEIGHTZERO v23, v3, v25, 3 // tile:7, oc:4-7
cbz x28, L8Tile8QuanUseInt8
cbz x27, TILE8_ADD_DSTV
ld1 {v2.4s, v3.4s}, [x27], x25
@@ -666,7 +664,7 @@ TILE8_BLOCKNUM:
MLA_WEIGHTZERO v15, v3, v24, 3 // tile:7, oc:0-3
MLA_WEIGHTZERO v16, v2, v25, 0 // tile:0, oc:4-7
MLA_WEIGHTZERO v17, v2, v25, 1 // tile:1, oc:4-7
MLA_WEIGHTZERO v18, v2, v25, 2 // tile:2, oc:4-7
MLA_WEIGHTZERO v18, v2, v25, 2 // tile:2, oc:4-7
MLA_WEIGHTZERO v19, v2, v25, 3 // tile:3, oc:4-7
MLA_WEIGHTZERO v20, v3, v25, 0 // tile:4, oc:4-7
MLA_WEIGHTZERO v21, v3, v25, 1 // tile:5, oc:4-7
@@ -697,6 +695,7 @@ TILE8_BLOCKNUM:
b TILE8_BLOCKNUM
TILE8_POST:
cbz x28, L8Tile8QuanUseInt8
sub x14, x14, #2 // oc-2
cbz x9, TILE8_RELU
ld1 {v0.4s, v1.4s}, [x20], #32
@@ -763,7 +762,7 @@ TILE8_BLOCKNUM:
cbz x27, L8LoopDz_TILE_8
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
b L8LoopDz_TILE_8
L4LoopDz_TILE_8:
mov x11, x1
mov x19, #0
@@ -803,7 +802,7 @@ L4_TILE8_BLOCKNUM:
ld1 {v4.4s, v5.4s}, [x24], x25
MUL_EXTRA_SCALE v4, v8, v9, v10, v11
MUL_EXTRA_SCALE v5, v12, v13, v14, v15
TILE8_L4_MLA:
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
@@ -813,7 +812,6 @@ L4_TILE8_BLOCKNUM:
MLA_WEIGHTZERO v13, v3, v24, 1 // tile:5, oc:0-3
MLA_WEIGHTZERO v14, v3, v24, 2 // tile:6, oc:0-3
MLA_WEIGHTZERO v15, v3, v24, 3 // tile:7, oc:0-3
cbz x28, L4Tile8QuanUseInt8
cbz x27, L4_TILE8_ADD_DSTV
ld1 {v2.4s, v3.4s}, [x27], x25
@@ -846,6 +844,7 @@ L4_TILE8_BLOCKNUM:
b L4_TILE8_BLOCKNUM
L4_TILE8_POST:
cbz x28, L4Tile8QuanUseInt8
cbz x9, L4_TILE8_RELU
ld1 {v0.4s}, [x20], #16
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
@@ -965,7 +964,7 @@ TILE4_BLOCKNUM:
MLA_WEIGHTZERO v13, v2, v25, 1 // tile:1, oc:4-7
MLA_WEIGHTZERO v14, v2, v25, 2 // tile:2, oc:4-7
MLA_WEIGHTZERO v15, v2, v25, 3 // tile:3, oc:4-7
cbz x28, L8Tile4QuanUseInt8
cbz x27, TILE4_ADD_DSTV
ld1 {v2.4s}, [x27], x25
@@ -997,6 +996,7 @@ TILE4_BLOCKNUM:
b TILE4_BLOCKNUM
TILE4_POST:
cbz x28, L8Tile4QuanUseInt8
sub x14, x14, #2
cbz x9, TILE4_RELU
ld1 {v4.4s, v5.4s}, [x20], #32
@@ -1082,7 +1082,6 @@ L4_TILE4_BLOCKNUM:
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
cbz x28, L4Tile4QuanUseInt8
cbz x27, L4_TILE4_ADD_DSTV
ld1 {v2.4s}, [x27], x25
@@ -1106,6 +1105,7 @@ L4_TILE4_BLOCKNUM:
b L4_TILE4_BLOCKNUM
L4_TILE4_POST:
cbz x28, L4Tile4QuanUseInt8
cbz x9, L4_TILE4_RELU
ld1 {v4.4s}, [x20], #16
ADD_BIAS_FLOAT v8, v9, v10, v11, v4
@@ -1172,7 +1172,7 @@ TILE_1_Init_FLOAT:
cbz x23, TILE_1
ld1r {v26.4s}, [x23], #4 // f32 min
ld1r {v27.4s}, [x23] // f32 max
TILE_1:
mov x6, x0
mov x12, x2
@@ -1216,7 +1216,7 @@ TILE1_BLOCKNUM:
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
MLA_WEIGHTZERO v9, v2, v25, 0 // tile:0, oc:4-7
cbz x28, L8Tile1QuanUseInt8
cbz x27, TILE1_ADD_DSTV
ld1 {v2.s}[0], [x27], x25
@@ -1238,6 +1238,7 @@ TILE1_BLOCKNUM:
b TILE1_BLOCKNUM
TILE1_POST:
cbz x28, L8Tile1QuanUseInt8
sub x14, x14, #2
cbz x9, TILE1_RELU
ld1 {v10.4s, v11.4s}, [x20], #32
@@ -1308,7 +1309,6 @@ L4_TILE1_BLOCKNUM:
TILE1_L4_MLA:
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
cbz x28, L4Tile1QuanUseInt8
cbz x27, L4_TILE1_ADD_DSTV
ld1 {v2.s}[0], [x27], x25
@@ -1328,7 +1328,10 @@ L4_TILE1_BLOCKNUM:
st1 {v8.4s}, [x10]
b L4_TILE1_BLOCKNUM
L4_TILE1_POST:
cbz x28, L4Tile1QuanUseInt8
cbz x9, L4_TILE1_RELU
ld1 {v10.4s}, [x20], #16
fadd v8.4s, v8.4s, v10.4s
source/backend/cpu/arm/arm64/MNNGemmInt8AddBiasScale_ARMV86_Unit.S
+25 -26
View File
@@ -109,7 +109,7 @@ mul \rg1, \rg2, \rg1
sub \rg0, \rg0, \rg1, LSL #5 // revert weight kernel sum
.endm
asm_function MNNGemmInt8AddBiasScale_ARMV86_Unit
/*
/*
struct QuanPostTreatParameters {
const float* scale;
const float* biasFloat;
@@ -172,9 +172,9 @@ TILE_10:
mov x20, x9 // bias
sub x4, x4, #32 // For int8 output, x4-32
mov x6, x27 // input bias
mov x12, #-256
cbz x28, LoopDz8_TILE_10
sub x4, x4, #96 // For int8 output, x4-32-96
mov x12, #-256
LoopDz8_TILE_10:
cmp x5, #2
@@ -193,7 +193,7 @@ L8_TILE10_BLOCKNUM:
LoopSz_TILE_10:
ld1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #64 // weight
ld1 {v3.16b, v4.16b, v5.16b, v6.16b}, [x11], #64 // src: E0-E9
ld1 {v7.16b}, [x11], #16
ld1 {v7.16b}, [x11], #16
subs x13, x13, #1
.inst 0x4e88a46c // smmla v12.4s, v3.16b, v8.16b // tile0-oc0, tile0-oc1, tile1-oc0, tile1-oc1
.inst 0x4e89a46d // smmla v13.4s, v3.16b, v9.16b // tile0-oc2, tile0-oc3, tile1-oc2, tile1-oc3
@@ -204,7 +204,7 @@ LoopSz_TILE_10:
.inst 0x4e89a491 // smmla v17.4s, v4.16b, v9.16b // tile2-oc2, tile2-oc3, tile3-oc2, tile3-oc3
.inst 0x4e8aa492 // smmla v18.4s, v4.16b, v10.16b // tile2-oc4, tile2-oc5, tile3-oc4, tile3-oc5
.inst 0x4e8ba493 // smmla v19.4s, v4.16b, v11.16b // tile2-oc6, tile2-oc7, tile3-oc6, tile3-oc7
.inst 0x4e88a4b4 // smmla v20.4s, v5.16b, v8.16b // tile4-oc0, tile4-oc1, tile5-oc0, tile5-oc1
.inst 0x4e89a4b5 // smmla v21.4s, v5.16b, v9.16b // tile4-oc2, tile4-oc3, tile5-oc2, tile5-oc3
.inst 0x4e8aa4b6 // smmla v22.4s, v5.16b, v10.16b // tile4-oc4, tile4-oc5, tile5-oc4, tile5-oc5
@@ -302,7 +302,6 @@ Tile10Quan:
MLA_WEIGHTZERO v18, v24, v26, 0 // tile:8, oc:4-7
MLA_WEIGHTZERO v19, v24, v26, 1 // tile:9, oc:4-7
cbz x28, Tile10QuanUseInt8
cbz x6, TILE10_ADD_DSTV
ld1 {v22.4s, v23.4s}, [x27], #32 // input dequant bias
@@ -359,6 +358,7 @@ Tile10Quan:
b L8_TILE10_BLOCKNUM
TILE10_POST:
cbz x28, Tile10QuanUseInt8
sub x5, x5, #2
cbz x9, L8_TILE10_RELU
ld1 {v20.4s, v21.4s}, [x20], #32 // bias
@@ -472,10 +472,10 @@ L4_LoopSz_TILE_10:
add x2, x2, #64 // x2+lp*hp
.inst 0x4e88a46c // smmla v12.4s, v3.16b, v8.16b // tile0-oc0, tile0-oc1, tile1-oc0, tile1-oc1
.inst 0x4e89a46d // smmla v13.4s, v3.16b, v9.16b // tile0-oc2, tile0-oc3, tile1-oc2, tile1-oc3
.inst 0x4e88a490 // smmla v16.4s, v4.16b, v8.16b // tile2-oc0, tile2-oc1, tile3-oc0, tile3-oc1
.inst 0x4e89a491 // smmla v17.4s, v4.16b, v9.16b // tile2-oc2, tile2-oc3, tile3-oc2, tile3-oc3
.inst 0x4e88a4b4 // smmla v20.4s, v5.16b, v8.16b // tile4-oc0, tile4-oc1, tile5-oc0, tile5-oc1
.inst 0x4e89a4b5 // smmla v21.4s, v5.16b, v9.16b // tile4-oc2, tile4-oc3, tile5-oc2, tile5-oc3
@@ -514,7 +514,7 @@ L4_Tile10Quan:
MUL_SCALE v20, v4, v5, v6, v7
fmul v8.4s, v8.4s, v20.4s
fmul v9.4s, v9.4s, v20.4s
cbz x21, L4_TILE10_MLA
ld1 {v27.4s, v28.4s}, [x23], #32 // input dequant scale
ld1 {v29.d}[0], [x23], x24
@@ -535,8 +535,6 @@ L4_Tile10Quan:
MLA_WEIGHTZERO v8, v24, v25, 0 // tile:8, oc:0-3
MLA_WEIGHTZERO v9, v24, v25, 1 // tile:9, oc:0-3
cbz x28, Tile10QuanUseInt8_L4
cbz x27, L4_TILE10_ADD_DSTV
ld1 {v22.4s, v23.4s}, [x27], #32 // input dequant bias
ld1 {v24.2s}, [x27], #8
@@ -575,6 +573,7 @@ L4_Tile10Quan:
b L4_TILE10_BLOCKNUM
L4_TILE10_POST:
cbz x28, Tile10QuanUseInt8_L4
cbz x9, L4_TILE10_RELU
ld1 {v20.4s}, [x20] // bias
ADD_BIAS_FLOAT v0, v1, v2, v3, v20
@@ -684,12 +683,12 @@ LoopSz_TILE_8:
.inst 0x4e89a46d // smmla v13.4s, v3.16b, v9.16b // tile0-oc2, tile0-oc3, tile1-oc2, tile1-oc3
.inst 0x4e8aa46e // smmla v14.4s, v3.16b, v10.16b // tile0-oc4, tile0-oc5, tile1-oc4, tile1-oc5
.inst 0x4e8ba46f // smmla v15.4s, v3.16b, v11.16b // tile0-oc6, tile0-oc7, tile1-oc6, tile1-oc7
.inst 0x4e88a490 // smmla v16.4s, v4.16b, v8.16b // tile2-oc0, tile2-oc1, tile3-oc0, tile3-oc1
.inst 0x4e89a491 // smmla v17.4s, v4.16b, v9.16b // tile2-oc2, tile2-oc3, tile3-oc2, tile3-oc3
.inst 0x4e8aa492 // smmla v18.4s, v4.16b, v10.16b // tile2-oc4, tile2-oc5, tile3-oc4, tile3-oc5
.inst 0x4e8ba493 // smmla v19.4s, v4.16b, v11.16b // tile2-oc6, tile2-oc7, tile3-oc6, tile3-oc7
.inst 0x4e88a4b4 // smmla v20.4s, v5.16b, v8.16b // tile4-oc0, tile4-oc1, tile5-oc0, tile5-oc1
.inst 0x4e89a4b5 // smmla v21.4s, v5.16b, v9.16b // tile4-oc2, tile4-oc3, tile5-oc2, tile5-oc3
.inst 0x4e8aa4b6 // smmla v22.4s, v5.16b, v10.16b // tile4-oc4, tile4-oc5, tile5-oc4, tile5-oc5
@@ -761,7 +760,6 @@ Tile8Quan:
MLA_WEIGHTZERO v14, v23, v26, 2
MLA_WEIGHTZERO v15, v23, v26, 3
cbz x28, Tile8QuanUseInt8
cbz x27, TILE8_ADD_DSTV
ld1 {v22.4s, v23.4s}, [x27], x25 // input dequant bias
@@ -808,6 +806,7 @@ Tile8Quan:
b L8_TILE8_BLOCKNUM
TILE8_POST:
cbz x28, Tile8QuanUseInt8
sub x24, x24, #2 // dz-2
cbz x9, TILE8_RELU
ld1 {v16.4s, v17.4s}, [x20], #32 // bias
@@ -868,7 +867,7 @@ Tile8LoopCheck:
cbz x27, LoopDz_TILE_8
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x25
b LoopDz_TILE_8
LoopDz4_TILE_8:
mov x11, x1 // src
mov x19, #0
@@ -885,10 +884,10 @@ L4_LoopSz_TILE_8:
add x12, x12, #64
.inst 0x4e88a46c // smmla v12.4s, v3.16b, v8.16b // tile0-oc0, tile0-oc1, tile1-oc0, tile1-oc1
.inst 0x4e89a46d // smmla v13.4s, v3.16b, v9.16b // tile0-oc2, tile0-oc3, tile1-oc2, tile1-oc3
.inst 0x4e88a490 // smmla v16.4s, v4.16b, v8.16b // tile2-oc0, tile2-oc1, tile3-oc0, tile3-oc1
.inst 0x4e89a491 // smmla v17.4s, v4.16b, v9.16b // tile2-oc2, tile2-oc3, tile3-oc2, tile3-oc3
.inst 0x4e88a4b4 // smmla v20.4s, v5.16b, v8.16b // tile4-oc0, tile4-oc1, tile5-oc0, tile5-oc1
.inst 0x4e89a4b5 // smmla v21.4s, v5.16b, v9.16b // tile4-oc2, tile4-oc3, tile5-oc2, tile5-oc3
@@ -933,7 +932,6 @@ L4_Tile8Quan:
MLA_WEIGHTZERO v6, v23, v25, 2
MLA_WEIGHTZERO v7, v23, v25, 3
cbz x28, Tile8QuanUseInt8_L4
cbz x27, L4_TILE8_ADD_DSTV
ld1 {v22.4s, v23.4s}, [x27], x25 // input dequant bias
@@ -966,6 +964,7 @@ L4_Tile8Quan:
b L4_TILE8_BLOCKNUM
L4_TILE8_POST:
cbz x28, Tile8QuanUseInt8_L4
cbz x9, L4_TILE8_RELU
ld1 {v16.4s}, [x20], #16 // bias
ADD_BIAS_FLOAT v0, v1, v2, v3, v16
@@ -1051,7 +1050,7 @@ LoopSz_TILE_4:
.inst 0x4e89a48d // smmla v13.4s, v4.16b, v9.16b // tile0-oc2, tile0-oc3, tile1-oc2, tile1-oc3
.inst 0x4e8aa48e // smmla v14.4s, v4.16b, v10.16b // tile0-oc4, tile0-oc5, tile1-oc4, tile1-oc5
.inst 0x4e8ba48f // smmla v15.4s, v4.16b, v11.16b // tile0-oc6, tile0-oc7, tile1-oc6, tile1-oc7
.inst 0x4e88a4b0 // smmla v16.4s, v5.16b, v8.16b // tile2-oc0, tile2-oc1, tile3-oc0, tile3-oc1
.inst 0x4e89a4b1 // smmla v17.4s, v5.16b, v9.16b // tile2-oc2, tile2-oc3, tile3-oc2, tile3-oc3
.inst 0x4e8aa4b2 // smmla v18.4s, v5.16b, v10.16b // tile2-oc4, tile2-oc5, tile3-oc4, tile3-oc5
@@ -1092,7 +1091,6 @@ Tile4Quan:
MLA_WEIGHTZERO v6, v22, v26, 2 // tile:2, oc:4-7
MLA_WEIGHTZERO v7, v22, v26, 3 // tile:3, oc:4-7
cbz x28, Tile4QuanUseInt8
cbz x27, TILE4_ADD_DSTV
ld1 {v22.4s}, [x27], x25 // input dequant bias
@@ -1125,6 +1123,7 @@ Tile4Quan:
b TILE4_BLOCKNUM
TILE4_POST:
cbz x28, Tile4QuanUseInt8
sub x24, x24, #2
cbz x9, TILE4_RELU
ld1 {v16.4s, v17.4s}, [x20], #32 // bias
@@ -1180,7 +1179,7 @@ L4_LoopSz_TILE_4:
add x12, x12, #64
.inst 0x4e88a48c // smmla v12.4s, v4.16b, v8.16b // tile0-oc0, tile0-oc1, tile1-oc0, tile1-oc1
.inst 0x4e89a48d // smmla v13.4s, v4.16b, v9.16b // tile0-oc2, tile0-oc3, tile1-oc2, tile1-oc3
.inst 0x4e88a4b0 // smmla v16.4s, v5.16b, v8.16b // tile2-oc0, tile2-oc1, tile3-oc0, tile3-oc1
.inst 0x4e89a4b1 // smmla v17.4s, v5.16b, v9.16b // tile2-oc2, tile2-oc3, tile3-oc2, tile3-oc3
bne L4_LoopSz_TILE_4
@@ -1210,7 +1209,6 @@ L4_Tile4Quan:
MLA_WEIGHTZERO v2, v22, v25, 2 // tile:2, oc:0-3
MLA_WEIGHTZERO v3, v22, v25, 3 // tile:3, oc:0-3
cbz x28, Tile4QuanUseInt8_L4
cbz x27, L4_TILE4_ADD_DSTV
ld1 {v22.4s}, [x27], x25 // input dequant bias
@@ -1234,6 +1232,7 @@ L4_Tile4Quan:
b L4_TILE4_BLOCKNUM
L4_TILE4_POST:
cbz x28, Tile4QuanUseInt8_L4
cbz x9, L4_TILE4_RELU
ld1 {v16.4s}, [x20], #16 // bias
ADD_BIAS_FLOAT v0, v1, v2, v3, v16
@@ -1332,7 +1331,6 @@ Tile2Quan:
MLA_WEIGHTZERO v2, v22, v26, 0 // tile:0, oc:4-7
MLA_WEIGHTZERO v3, v22, v26, 1 // tile:1, oc:4-7
cbz x28, Tile2QuanUseInt8
cbz x27, TILE2_ADD_DSTV
ld1 {v22.2s}, [x27], x25 // input dequant bias
@@ -1355,6 +1353,7 @@ Tile2Quan:
b TILE2_BLOCKNUM
TILE2_POST:
cbz x28, Tile2QuanUseInt8
sub x24, x24, #2
cbz x9, TILE2_RELU
ld1 {v16.4s, v17.4s}, [x20], #32 // bias
@@ -1403,7 +1402,7 @@ Tile2LoopCheck:
cbz x27, LoopDz_TILE_2
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x25
bge LoopDz_TILE_2
LoopDz4_TILE_2:
mov x11, x1 // src
@@ -1445,7 +1444,6 @@ L4_Tile2Quan:
MLA_WEIGHTZERO v0, v22, v25, 0 // tile:0, oc:0-3
MLA_WEIGHTZERO v1, v22, v25, 1 // tile:1, oc:0-3
cbz x28, Tile2QuanUseInt8_L4
cbz x27, L4_TILE2_ADD_DSTV
ld1 {v22.2s}, [x27], x25 // input dequant bias
@@ -1468,6 +1466,7 @@ L4_Tile2Quan:
b L4_TILE2_BLOCKNUM
L4_TILE2_POST:
cbz x28, Tile2QuanUseInt8_L4
cbz x9, L4_TILE2_RELU
ld1 {v16.4s}, [x20], #16 // bias
fadd v0.4s, v0.4s, v16.4s
@@ -1565,7 +1564,6 @@ Tile1Quan:
MLA_WEIGHTZERO v25, v6, v8, 0 // tile:0, oc:0-3
MLA_WEIGHTZERO v26, v6, v9, 0 // tile:0, oc:4-7
cbz x28, Tile1QuanUseInt8
cbz x27, TILE1_ADD_DSTV
ld1 {v6.s}[0], [x27], x25 // input dequant bias
@@ -1587,6 +1585,7 @@ Tile1Quan:
b TILE1_BLOCKNUM
TILE1_POST:
cbz x28, Tile1QuanUseInt8
sub x24, x24, #2
cbz x9, TILE1_RELU
ld1 {v16.4s, v17.4s}, [x20], #32 // bias
@@ -1666,7 +1665,6 @@ L4_Tile1Quan:
L4_TILE1_MLA:
MLA_WEIGHTZERO v25, v6, v8, 0 // tile:0, oc:0-3
cbz x28, Tile1QuanUseInt8_L4
cbz x27, L4_TILE1_ADD_DSTV
ld1 {v6.s}[0], [x27], x25 // input dequant bias
@@ -1687,6 +1685,7 @@ L4_Tile1Quan:
b L4_TILE1_BLOCKNUM
L4_TILE1_POST:
cbz x28, Tile1QuanUseInt8_L4
cbz x9, L4_TILE1_RELU
ld1 {v16.4s}, [x20], #16 // bias
fadd v25.4s, v25.4s, v16.4s
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScale16x32_SME2_w4_Fp16.S
+189 -1259
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScale16x32_SME2_w4_Fp32.S
+118 -1412
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScale16x32_SME2_w8_Fp16.S
+190 -1249
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScale16x32_SME2_w8_Fp32.S
+220 -2140
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScaleHp128_SME2_w4_Fp16.S
+163 -162
View File
@@ -1,20 +1,20 @@
// MNNGemmInt8AddBiasScaleHp128_SME2_w4_Fp16.S
// Created by MNN on 2022/09/26.
// Copyright © 2018, Alibaba Group Holding Limited
#if defined(__aarch64__)
#include "MNNAsmGlobal.h"
.text
.macro REVERT_INPUT_DEQUANT_BIAS rg0, rg1, rg2, rg3
mul \rg1, \rg2, \rg3
sub \rg0, \rg0, \rg1
.endm
asm_function MNNGemmInt8AddBiasScaleHp128_SME2_w4_Fp16
/*
/*
struct QuanPostTreatParameters {
const float* scale;
const float* biasFloat;
@@ -34,8 +34,8 @@ struct QuanPostTreatParameters {
//void MNNGemmInt8AddBiasScaleHp128_SME2_w4_Fp16(int8_t* dst, const int8_t* src,
// const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad,
// const QuanPostTreatParameters* parameters, size_t realDstCount);
//Auto: x0:dst, x1:src, x2:weight, x3:src_depth_quad, x4:dst_step x5:dst_depth_quad, x6: parameters, x7: realDstCount
//Auto: x0:dst, x1:src, x2:weight, x3:src_depth_quad, x4:dst_step x5:dst_depth_quad, x0: parameters, x7: realDstCount
// sme2 Ep=16, LP=4, HP=16
stp x29, x30, [sp, #-320]!
@@ -52,6 +52,7 @@ stp d14, d15, [sp, #32]
.inst 0xd503477f // smstart
ldr x9, [x6, #8] // biasFloat
ldr x13, [x6, #40] // srcKernelSum
ldr x28, [x6, #48] // weightKernelSum
@@ -67,33 +68,37 @@ ldr x14, [x6, #56] // float32 maxmin ptr
mov x22, #48
.inst 0x2518e080 // ptrue p0.b, #4 // first 4 bytes
.inst 0x2518e125 // ptrue p5.b, vl16 // first 16 bytes
.inst 0x2518e3e3 // ptrue p3.b // 64 bytes valid
.inst 0x25207810 // ptrue pn8.b
.inst 0x2598e3e1 // ptrue p1.s
.inst 0x2558e3e2 // ptrue p2.h
lsl x22, x7, #2 // eDest * GEMM_INT8_SRC_UNIT
mov x25, 0 // inputBlockNum=1
cbz x27, TILE_1
mov x25, x22 // input block quant: realDstCount * sizeof(float)
TILE_1:
cmp x7, #1
blt End
mov x24, x5 // dst_depth_quad
mov x6, x0 // dst
mov x12, x2 // weight
mov x20, x9 // bias
mov x19, x13 // input kernel sum
mov x21, x23 // input dequant scale
mov x24, x27
LoopDz_TILE1:
// load bias
lsl x10, x5, #3
mov x11, #64
.inst 0x25aa67f1 // whilelt pn9.s, xzr, x10, vlx4
.inst 0x25aa6572 // whilelt pn10.s, x11, x10, vlx4
.inst 0xa040c698 // ld1w {z24.s-z27.s}, pn9/z, [x20]
.inst 0xa041ca9c // ld1w {z28.s-z31.s}, pn10/z, [x20, #4, MUL VL]
.inst 0x04345114 // addvl x20, x20, #8
mov x11, x1 // src
mov x15, x26
.inst 0xa040c298 // ld1w {z24.s-z27.s}, pn8/z, [x20] // bias
.inst 0xa041c29c // ld1w {z28.s-z31.s}, pn8/z, [x20, #4, MUL VL] // bias
.inst 0x04345114 // addvl x20, x20, #8
TILE1_BLOCKNUM:
mov x10, x3 // src_depth_quad
@@ -113,20 +118,20 @@ TILE1_BLOCKNUM:
subs x10, x10, #1
add x11, x11, x22
.inst 0x042c508c // addvl x12, x12, #4
bne LoopSz_TILE_1_lu1
LoopSzEnd_TILE_1:
sub x15, x15, #1
.inst 0xa040c180 // ld1w {z0.s-z3.s}, pn8/z, [x12] // weight scale: 64*sizeof(float)
.inst 0xa041c190 // ld1w {z16.s-z19.s}, pn8/z, [x12, #4, MUL VL] // weight scale
.inst 0xa042c188 // ld1w {z8.s-z11.s}, pn8/z, [x12, #8, MUL VL] // weight bias
.inst 0xa043c18c // ld1w {z12.s-z15.s}, pn8/z, [x12, #12, MUL VL] // weight bias
.inst 0xa0408180 // ld1b {z0.b-z3.b}, pn8/z, [x12] // weight scale: 64*sizeof(float)
.inst 0xa0418190 // ld1b {z16.b-z19.b}, pn8/z, [x12, #4, MUL VL] // weight scale
.inst 0xa0428188 // ld1b {z8.b-z11.b}, pn8/z, [x12, #8, MUL VL] // weight bias
.inst 0xa043818c // ld1b {z12.b-z15.b}, pn8/z, [x12, #12, MUL VL] // weight bias
.inst 0x8540cda4 // ld1rw {z4.s}, p3/z, [x13] // x kernel sum
.inst 0x8540cef7 // ld1rw {z23.s}, p3/z, [x23] // x scale
.inst 0x8540c5a4 // ld1rw {z4.s}, p1/z, [x13] // x kernel sum
.inst 0x8540c6f7 // ld1rw {z23.s}, p1/z, [x23] // x scale
// input kernel sum x weight quant bias
.inst 0x64a40118 // fmla z24.s, z8.s, z4.s[0]
@@ -154,30 +159,30 @@ LoopSzEnd_TILE_1:
.inst 0xc0060c08 // mova {z8.s-z11.s}, za.s[w8, 0, VGx4]
.inst 0xc0060c8c // mova {z12.s-z15.s}, za.s[w8, 4, VGx4]
.inst 0x6594ad08 // scvtf z8.s, p3/m, z8.s
.inst 0x6594ad29 // scvtf z9.s, p3/m, z9.s
.inst 0x6594ad4a // scvtf z10.s, p3/m, z10.s
.inst 0x6594ad6b // scvtf z11.s, p3/m, z11.s
.inst 0x6594ad8c // scvtf z12.s, p3/m, z12.s
.inst 0x6594adad // scvtf z13.s, p3/m, z13.s
.inst 0x6594adce // scvtf z14.s, p3/m, z14.s
.inst 0x6594adef // scvtf z15.s, p3/m, z15.s
.inst 0x6594a508 // scvtf z8.s, p1/m, z8.s
.inst 0x6594a529 // scvtf z9.s, p1/m, z9.s
.inst 0x6594a54a // scvtf z10.s, p1/m, z10.s
.inst 0x6594a56b // scvtf z11.s, p1/m, z11.s
.inst 0x6594a58c // scvtf z12.s, p1/m, z12.s
.inst 0x6594a5ad // scvtf z13.s, p1/m, z13.s
.inst 0x6594a5ce // scvtf z14.s, p1/m, z14.s
.inst 0x6594a5ef // scvtf z15.s, p1/m, z15.s
// accumulate
.inst 0x65a00d18 // fmla z24.s, p3/m, z8.s, z0.s
.inst 0x65a10d39 // fmla z25.s, p3/m, z9.s, z1.s
.inst 0x65a20d5a // fmla z26.s, p3/m, z10.s, z2.s
.inst 0x65a30d7b // fmla z27.s, p3/m, z11.s, z3.s
.inst 0x65b00d9c // fmla z28.s, p3/m, z12.s, z16.s
.inst 0x65b10dbd // fmla z29.s, p3/m, z13.s, z17.s
.inst 0x65b20dde // fmla z30.s, p3/m, z14.s, z18.s
.inst 0x65b30dff // fmla z31.s, p3/m, z15.s, z19.s
.inst 0x65a00518 // fmla z24.s, p1/m, z8.s, z0.s
.inst 0x65a10539 // fmla z25.s, p1/m, z9.s, z1.s
.inst 0x65a2055a // fmla z26.s, p1/m, z10.s, z2.s
.inst 0x65a3057b // fmla z27.s, p1/m, z11.s, z3.s
.inst 0x65b0059c // fmla z28.s, p1/m, z12.s, z16.s
.inst 0x65b105bd // fmla z29.s, p1/m, z13.s, z17.s
.inst 0x65b205de // fmla z30.s, p1/m, z14.s, z18.s
.inst 0x65b305ff // fmla z31.s, p1/m, z15.s, z19.s
.inst 0x042c520c // addvl x12, x12, #16 // weight ptr
add x13, x13, x22
cbz x27, TILE1_ADD_DSTV
.inst 0x8540cf64 // ld1rw {z4.s}, p3/z, [x27] // input dequant bias
.inst 0x8540c764 // ld1rw {z4.s}, p1/z, [x27] // input dequant bias
.inst 0xa0408388 // ld1b {z8.b-z11.b}, pn8/z, [x28] // weight kernel sum
.inst 0xa041838c // ld1b {z12.b-z15.b}, pn8/z, [x28, #4, MUL VL] // weight kernel sum
.inst 0x64a40118 // fmla z24.s, z8.s, z4.s[0]
@@ -188,8 +193,8 @@ LoopSzEnd_TILE_1:
.inst 0x64a401bd // fmla z29.s, z13.s, z4.s[0]
.inst 0x64a401de // fmla z30.s, z14.s, z4.s[0]
.inst 0x64a401ff // fmla z31.s, z15.s, z4.s[0]
add x27, x27, x25
add x23, x23, x25
add x27, x27, x22
add x23, x23, x22
.inst 0x043c511c // addvl x28, x28, #8
TILE1_ADD_DSTV:
@@ -197,47 +202,47 @@ LoopSzEnd_TILE_1:
bne TILE1_BLOCKNUM
TILE1_STORE:
.inst 0x84c0adc8 // ld1rh {z8.h}, p3/z, [x14]
.inst 0x84c1adc9 // ld1rh {z9.h}, p3/z, [x14, #2]
.inst 0x84c0a9c8 // ld1rh {z8.h}, p2/z, [x14]
.inst 0x84c1a9c9 // ld1rh {z9.h}, p2/z, [x14, #2]
.inst 0xc120e314 // fcvt z20.h, {z24.s-z25.s}
.inst 0xc120e355 // fcvt z21.h, {z26.s-z27.s}
.inst 0xc120e396 // fcvt z22.h, {z28.s-z29.s}
.inst 0xc120e3d7 // fcvt z23.h, {z30.s-z31.s}
.inst 0xc169c914 // fclamp {z20.h-z23.h}, z8.h, z9.h
cmp x24, #16
cmp x5, #16
bge TILE1_STORE128
cmp x24, #1
cmp x5, #1
beq TILE1_STORE8
cmp x24, #2
cmp x5, #2
beq TILE1_STORE16
cmp x24, #3
cmp x5, #3
beq TILE1_STORE24
cmp x24, #4
cmp x5, #4
beq TILE1_STORE32
cmp x24, #5
cmp x5, #5
beq TILE1_STORE40
cmp x24, #6
cmp x5, #6
beq TILE1_STORE48
cmp x24, #7
cmp x5, #7
beq TILE1_STORE56
cmp x24, #8
cmp x5, #8
beq TILE1_STORE64
cmp x24, #9
cmp x5, #9
beq TILE1_STORE72
cmp x24, #10
cmp x5, #10
beq TILE1_STORE80
cmp x24, #11
cmp x5, #11
beq TILE1_STORE88
cmp x24, #12
cmp x5, #12
beq TILE1_STORE96
cmp x24, #13
cmp x5, #13
beq TILE1_STORE104
cmp x24, #14
cmp x5, #14
beq TILE1_STORE112
cmp x24, #15
cmp x5, #15
beq TILE1_STORE120
@@ -253,15 +258,15 @@ LoopSzEnd_TILE_1:
.inst 0x05f022c8 // dup z8.q, z22.q[3]
.inst 0x057022e9 // dup z9.q, z23.q[1]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -274,7 +279,7 @@ LoopSzEnd_TILE_1:
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
.inst 0xe40456e9 // st1b {z9.b}, p5, [x23, x4]
b TILE1_Dz_End
b End
TILE1_STORE104: // M = 13
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -287,15 +292,15 @@ LoopSzEnd_TILE_1:
.inst 0x05b022c7 // dup z7.q, z22.q[2]
.inst 0x05f022c8 // dup z8.q, z22.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -307,7 +312,7 @@ LoopSzEnd_TILE_1:
.inst 0xe400f5a7 // st1b {z7.b}, p5, [x13]
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
b TILE1_Dz_End
b End
TILE1_STORE96: // M = 12
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -320,14 +325,14 @@ LoopSzEnd_TILE_1:
.inst 0x05b022c7 // dup z7.q, z22.q[2]
.inst 0x05f022c8 // dup z8.q, z22.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -338,7 +343,7 @@ LoopSzEnd_TILE_1:
.inst 0xe4045566 // st1b {z6.b}, p5, [x11, x4]
.inst 0xe400f5a7 // st1b {z7.b}, p5, [x13]
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
b TILE1_Dz_End
b End
TILE1_STORE88: // M = 11
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -350,14 +355,14 @@ LoopSzEnd_TILE_1:
.inst 0x057022c6 // dup z6.q, z22.q[1]
.inst 0x05b022c7 // dup z7.q, z22.q[2]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -367,7 +372,7 @@ LoopSzEnd_TILE_1:
.inst 0xe400f576 // st1b {z22.b}, p5, [x11]
.inst 0xe4045566 // st1b {z6.b}, p5, [x11, x4]
.inst 0xe400f5a7 // st1b {z7.b}, p5, [x13]
b TILE1_Dz_End
b End
TILE1_STORE80: // M = 10
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -378,13 +383,13 @@ LoopSzEnd_TILE_1:
.inst 0x05f022a5 // dup z5.q, z21.q[3]
.inst 0x057022c6 // dup z6.q, z22.q[1]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -393,7 +398,7 @@ LoopSzEnd_TILE_1:
.inst 0xe4045505 // st1b {z5.b}, p5, [x8, x4]
.inst 0xe400f576 // st1b {z22.b}, p5, [x11]
.inst 0xe4045566 // st1b {z6.b}, p5, [x11, x4]
b TILE1_Dz_End
b End
TILE1_STORE72: // M = 9
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -403,13 +408,13 @@ LoopSzEnd_TILE_1:
.inst 0x05b022a4 // dup z4.q, z21.q[2]
.inst 0x05f022a5 // dup z5.q, z21.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -417,7 +422,7 @@ LoopSzEnd_TILE_1:
.inst 0xe400f504 // st1b {z4.b}, p5, [x8]
.inst 0xe4045505 // st1b {z5.b}, p5, [x8, x4]
.inst 0xe400f576 // st1b {z22.b}, p5, [x11]
b TILE1_Dz_End
b End
TILE1_STORE64: // M = 8
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -427,19 +432,19 @@ LoopSzEnd_TILE_1:
.inst 0x05b022a4 // dup z4.q, z21.q[2]
.inst 0x05f022a5 // dup z5.q, z21.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
.inst 0xe40455e3 // st1b {z3.b}, p5, [x15, x4]
.inst 0xe400f504 // st1b {z4.b}, p5, [x8]
.inst 0xe4045505 // st1b {z5.b}, p5, [x8, x4]
b TILE1_Dz_End
b End
TILE1_STORE56: // M = 7
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -448,18 +453,18 @@ LoopSzEnd_TILE_1:
.inst 0x057022a3 // dup z3.q, z21.q[1]
.inst 0x05b022a4 // dup z4.q, z21.q[2]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
.inst 0xe40455e3 // st1b {z3.b}, p5, [x15, x4]
.inst 0xe400f504 // st1b {z4.b}, p5, [x8]
b TILE1_Dz_End
b End
TILE1_STORE48: // M = 6
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -467,66 +472,66 @@ LoopSzEnd_TILE_1:
.inst 0x05f02282 // dup z2.q, z20.q[3]
.inst 0x057022a3 // dup z3.q, z21.q[1]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
.inst 0xe40455e3 // st1b {z3.b}, p5, [x15, x4]
b TILE1_Dz_End
b End
TILE1_STORE40: // M = 5
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0x05b02281 // dup z1.q, z20.q[2]
.inst 0x05f02282 // dup z2.q, z20.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
b TILE1_Dz_End
b End
TILE1_STORE32: // M = 4
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0x05b02281 // dup z1.q, z20.q[2]
.inst 0x05f02282 // dup z2.q, z20.q[3]
add x9, x6, x4, lsl #1 // +2
add x9, x0, x4, lsl #1 // +2
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
b TILE1_Dz_End
b End
TILE1_STORE24: // M = 3
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0x05b02281 // dup z1.q, z20.q[2]
add x9, x6, x4, lsl #1 // +2
add x9, x0, x4, lsl #1 // +2
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
b TILE1_Dz_End
b End
TILE1_STORE16: // M = 2
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
b TILE1_Dz_End
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
b End
TILE1_STORE8: // M = 1
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
b TILE1_Dz_End
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
b End
TILE1_STORE120:
@@ -542,16 +547,16 @@ LoopSzEnd_TILE_1:
.inst 0x057022e9 // dup z9.q, z23.q[1]
.inst 0x05b022ea // dup z10.q, z23.q[2]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
add x5, x8, x4, lsl #3 // +14
add x6, x8, x4, lsl #3 // +14
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -564,9 +569,9 @@ LoopSzEnd_TILE_1:
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
.inst 0xe40456e9 // st1b {z9.b}, p5, [x23, x4]
.inst 0xe400f4aa // st1b {z10.b}, p5, [x5]
.inst 0xe400f4ca // st1b {z10.b}, p5, [x6]
b TILE1_Dz_End
b End
TILE1_STORE128:
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -582,16 +587,16 @@ LoopSzEnd_TILE_1:
.inst 0x05b022ea // dup z10.q, z23.q[2]
.inst 0x05f022eb // dup z11.q, z23.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
add x5, x8, x4, lsl #3 // +14
add x6, x8, x4, lsl #3 // +14
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -604,25 +609,21 @@ LoopSzEnd_TILE_1:
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
.inst 0xe40456e9 // st1b {z9.b}, p5, [x23, x4]
.inst 0xe400f4aa // st1b {z10.b}, p5, [x5]
.inst 0xe40454ab // st1b {z11.b}, p5, [x5, x4]
.inst 0xe400f4ca // st1b {z10.b}, p5, [x6]
.inst 0xe40454cb // st1b {z11.b}, p5, [x6, x4]
TILE1_Dz_End:
sub x24, x24, #16
cmp x24, #0
ble End
add x6, x6, x4, LSL #4
subs x5, x5, #16
add x0, x0, x4, LSL #4
beq End
mov x13, x19
mov x23, x21
cbz x27, LoopDz_TILE1
REVERT_INPUT_DEQUANT_BIAS x27, x15, x26, x25
mov x27, x24
b LoopDz_TILE1
End:
.inst 0xd503467f // smstop
ldp x19, x20, [sp, #224]
ldp x21, x22, [sp, #208]
ldp x23, x24, [sp, #192]
@@ -634,5 +635,5 @@ ldp d12, d13, [sp, #48]
ldp d14, d15, [sp, #32]
ldp x29, x30, [sp], #320
ret
#endif // __aarch64__
#endif // __aarch64__
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScaleHp128_SME2_w4_Fp32.S
+357 -360
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScaleHp128_SME2_w8_Fp16.S
+170 -169
View File
@@ -1,20 +1,20 @@
// MNNGemmInt8AddBiasScaleHp128_SME2_w8_Fp16.S
// Created by MNN on 2022/09/26.
// Copyright © 2018, Alibaba Group Holding Limited
#if defined(__aarch64__)
#include "MNNAsmGlobal.h"
.text
.macro REVERT_INPUT_DEQUANT_BIAS rg0, rg1, rg2, rg3
mul \rg1, \rg2, \rg3
sub \rg0, \rg0, \rg1
.endm
asm_function MNNGemmInt8AddBiasScaleHp128_SME2_w8_Fp16
/*
/*
struct QuanPostTreatParameters {
const float* scale;
const float* biasFloat;
@@ -34,10 +34,10 @@ struct QuanPostTreatParameters {
//void MNNGemmInt8AddBiasScaleHp128_SME2_w8_Fp16(int8_t* dst, const int8_t* src,
// const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad,
// const QuanPostTreatParameters* parameters, size_t realDstCount);
//Auto: x0:dst, x1:src, x2:weight, x3:src_depth_quad, x4:dst_step x5:dst_depth_quad, x6: parameters, x7: realDstCount
//Auto: x0:dst, x1:src, x2:weight, x3:src_depth_quad, x4:dst_step x5:dst_depth_quad, x0: parameters, x7: realDstCount
// sme2 Ep=16, LP=4, HP=16
stp x29, x30, [sp, #-320]!
mov x29, sp
stp x19, x20, [sp, #224]
@@ -50,8 +50,8 @@ stp d10, d11, [sp, #64]
stp d12, d13, [sp, #48]
stp d14, d15, [sp, #32]
.inst 0xd503477f // smstart
ldr x9, [x6, #8] // biasFloat
ldr x13, [x6, #40] // srcKernelSum
ldr x28, [x6, #48] // weightKernelSum
@@ -60,38 +60,42 @@ ldr x23, [x6, #80] // input scale
ldr x27, [x6, #88] // input bias
ldr x8, [x6, #104] // indices
ldr x14, [x6, #56] // float32 maxmin ptr
/* initialize predicates */
mov x22, #48
.inst 0x2518e080 // ptrue p0.b, #4 // first 4 bytes
.inst 0x2518e125 // ptrue p5.b, vl16 // first 16 bytes
.inst 0x2518e3e3 // ptrue p3.b // 64 bytes valid
.inst 0x25207810 // ptrue pn8.b
.inst 0x2598e3e1 // ptrue p1.s
.inst 0x2558e3e2 // ptrue p2.h
lsl x22, x7, #2 // eDest * GEMM_INT8_SRC_UNIT
mov x25, 0 // inputBlockNum=1
cbz x27, TILE_1
mov x25, x22 // input block quant: realDstCount * sizeof(float)
TILE_1:
cmp x7, #1
blt End
mov x24, x5 // dst_depth_quad
mov x6, x0 // dst
mov x12, x2 // weight
mov x20, x9 // bias
mov x19, x13 // input kernel sum
mov x21, x23 // input dequant scale
mov x24, x27
LoopDz_TILE1:
// load bias
lsl x10, x5, #3
mov x11, #64
.inst 0x25aa67f1 // whilelt pn9.s, xzr, x10, vlx4
.inst 0x25aa6572 // whilelt pn10.s, x11, x10, vlx4
.inst 0xa040c698 // ld1w {z24.s-z27.s}, pn9/z, [x20]
.inst 0xa041ca9c // ld1w {z28.s-z31.s}, pn10/z, [x20, #4, MUL VL]
.inst 0x04345114 // addvl x20, x20, #8
mov x11, x1 // src
mov x15, x26
.inst 0xa040c298 // ld1w {z24.s-z27.s}, pn8/z, [x20] // bias
.inst 0xa041c29c // ld1w {z28.s-z31.s}, pn8/z, [x20, #4, MUL VL] // bias
.inst 0x04345114 // addvl x20, x20, #8
TILE1_BLOCKNUM:
mov x10, x3 // src_depth_quad
@@ -107,20 +111,20 @@ TILE1_BLOCKNUM:
subs x10, x10, #1
add x11, x11, x22
.inst 0x042c510c // addvl x12, x12, #8
bne LoopSz_TILE_1_lu1
LoopSzEnd_TILE_1:
sub x15, x15, #1
.inst 0xa040c180 // ld1w {z0.s-z3.s}, pn8/z, [x12] // weight scale: 64*sizeof(float)
.inst 0xa041c190 // ld1w {z16.s-z19.s}, pn8/z, [x12, #4, MUL VL] // weight scale
.inst 0xa042c188 // ld1w {z8.s-z11.s}, pn8/z, [x12, #8, MUL VL] // weight bias
.inst 0xa043c18c // ld1w {z12.s-z15.s}, pn8/z, [x12, #12, MUL VL] // weight bias
.inst 0xa0408180 // ld1b {z0.b-z3.b}, pn8/z, [x12] // weight scale: 64*sizeof(float)
.inst 0xa0418190 // ld1b {z16.b-z19.b}, pn8/z, [x12, #4, MUL VL] // weight scale
.inst 0xa0428188 // ld1b {z8.b-z11.b}, pn8/z, [x12, #8, MUL VL] // weight bias
.inst 0xa043818c // ld1b {z12.b-z15.b}, pn8/z, [x12, #12, MUL VL] // weight bias
.inst 0x8540cda4 // ld1rw {z4.s}, p3/z, [x13] // x kernel sum
.inst 0x8540cef7 // ld1rw {z23.s}, p3/z, [x23] // x scale
.inst 0x8540c5a4 // ld1rw {z4.s}, p1/z, [x13] // x kernel sum
.inst 0x8540c6f7 // ld1rw {z23.s}, p1/z, [x23] // x scale
// input kernel sum x weight quant bias
.inst 0x64a40118 // fmla z24.s, z8.s, z4.s[0]
@@ -148,30 +152,30 @@ LoopSzEnd_TILE_1:
.inst 0xc0060c08 // mova {z8.s-z11.s}, za.s[w8, 0, VGx4]
.inst 0xc0060c8c // mova {z12.s-z15.s}, za.s[w8, 4, VGx4]
.inst 0x6594ad08 // scvtf z8.s, p3/m, z8.s
.inst 0x6594ad29 // scvtf z9.s, p3/m, z9.s
.inst 0x6594ad4a // scvtf z10.s, p3/m, z10.s
.inst 0x6594ad6b // scvtf z11.s, p3/m, z11.s
.inst 0x6594ad8c // scvtf z12.s, p3/m, z12.s
.inst 0x6594adad // scvtf z13.s, p3/m, z13.s
.inst 0x6594adce // scvtf z14.s, p3/m, z14.s
.inst 0x6594adef // scvtf z15.s, p3/m, z15.s
.inst 0x6594a508 // scvtf z8.s, p1/m, z8.s
.inst 0x6594a529 // scvtf z9.s, p1/m, z9.s
.inst 0x6594a54a // scvtf z10.s, p1/m, z10.s
.inst 0x6594a56b // scvtf z11.s, p1/m, z11.s
.inst 0x6594a58c // scvtf z12.s, p1/m, z12.s
.inst 0x6594a5ad // scvtf z13.s, p1/m, z13.s
.inst 0x6594a5ce // scvtf z14.s, p1/m, z14.s
.inst 0x6594a5ef // scvtf z15.s, p1/m, z15.s
// accumulate
.inst 0x65a00d18 // fmla z24.s, p3/m, z8.s, z0.s
.inst 0x65a10d39 // fmla z25.s, p3/m, z9.s, z1.s
.inst 0x65a20d5a // fmla z26.s, p3/m, z10.s, z2.s
.inst 0x65a30d7b // fmla z27.s, p3/m, z11.s, z3.s
.inst 0x65b00d9c // fmla z28.s, p3/m, z12.s, z16.s
.inst 0x65b10dbd // fmla z29.s, p3/m, z13.s, z17.s
.inst 0x65b20dde // fmla z30.s, p3/m, z14.s, z18.s
.inst 0x65b30dff // fmla z31.s, p3/m, z15.s, z19.s
.inst 0x65a00518 // fmla z24.s, p1/m, z8.s, z0.s
.inst 0x65a10539 // fmla z25.s, p1/m, z9.s, z1.s
.inst 0x65a2055a // fmla z26.s, p1/m, z10.s, z2.s
.inst 0x65a3057b // fmla z27.s, p1/m, z11.s, z3.s
.inst 0x65b0059c // fmla z28.s, p1/m, z12.s, z16.s
.inst 0x65b105bd // fmla z29.s, p1/m, z13.s, z17.s
.inst 0x65b205de // fmla z30.s, p1/m, z14.s, z18.s
.inst 0x65b305ff // fmla z31.s, p1/m, z15.s, z19.s
.inst 0x042c520c // addvl x12, x12, #16 // weight ptr
add x13, x13, x22
cbz x27, TILE1_ADD_DSTV
.inst 0x8540cf64 // ld1rw {z4.s}, p3/z, [x27] // input dequant bias
.inst 0x8540c764 // ld1rw {z4.s}, p1/z, [x27] // input dequant bias
.inst 0xa0408388 // ld1b {z8.b-z11.b}, pn8/z, [x28] // weight kernel sum
.inst 0xa041838c // ld1b {z12.b-z15.b}, pn8/z, [x28, #4, MUL VL] // weight kernel sum
.inst 0x64a40118 // fmla z24.s, z8.s, z4.s[0]
@@ -182,56 +186,56 @@ LoopSzEnd_TILE_1:
.inst 0x64a401bd // fmla z29.s, z13.s, z4.s[0]
.inst 0x64a401de // fmla z30.s, z14.s, z4.s[0]
.inst 0x64a401ff // fmla z31.s, z15.s, z4.s[0]
add x27, x27, x25
add x23, x23, x25
add x27, x27, x22
add x23, x23, x22
.inst 0x043c511c // addvl x28, x28, #8
TILE1_ADD_DSTV:
cmp x15, #0
bne TILE1_BLOCKNUM
TILE1_STORE:
.inst 0x84c0adc8 // ld1rh {z8.h}, p3/z, [x14]
.inst 0x84c1adc9 // ld1rh {z9.h}, p3/z, [x14, #2]
.inst 0x84c0a9c8 // ld1rh {z8.h}, p2/z, [x14]
.inst 0x84c1a9c9 // ld1rh {z9.h}, p2/z, [x14, #2]
.inst 0xc120e314 // fcvt z20.h, {z24.s-z25.s}
.inst 0xc120e355 // fcvt z21.h, {z26.s-z27.s}
.inst 0xc120e396 // fcvt z22.h, {z28.s-z29.s}
.inst 0xc120e3d7 // fcvt z23.h, {z30.s-z31.s}
.inst 0xc169c914 // fclamp {z20.h-z23.h}, z8.h, z9.h
cmp x24, #16
cmp x5, #16
bge TILE1_STORE128
cmp x24, #1
cmp x5, #1
beq TILE1_STORE8
cmp x24, #2
cmp x5, #2
beq TILE1_STORE16
cmp x24, #3
cmp x5, #3
beq TILE1_STORE24
cmp x24, #4
cmp x5, #4
beq TILE1_STORE32
cmp x24, #5
cmp x5, #5
beq TILE1_STORE40
cmp x24, #6
cmp x5, #6
beq TILE1_STORE48
cmp x24, #7
cmp x5, #7
beq TILE1_STORE56
cmp x24, #8
cmp x5, #8
beq TILE1_STORE64
cmp x24, #9
cmp x5, #9
beq TILE1_STORE72
cmp x24, #10
cmp x5, #10
beq TILE1_STORE80
cmp x24, #11
cmp x5, #11
beq TILE1_STORE88
cmp x24, #12
cmp x5, #12
beq TILE1_STORE96
cmp x24, #13
cmp x5, #13
beq TILE1_STORE104
cmp x24, #14
cmp x5, #14
beq TILE1_STORE112
cmp x24, #15
cmp x5, #15
beq TILE1_STORE120
@@ -247,15 +251,15 @@ LoopSzEnd_TILE_1:
.inst 0x05f022c8 // dup z8.q, z22.q[3]
.inst 0x057022e9 // dup z9.q, z23.q[1]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -268,7 +272,7 @@ LoopSzEnd_TILE_1:
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
.inst 0xe40456e9 // st1b {z9.b}, p5, [x23, x4]
b TILE1_Dz_End
b End
TILE1_STORE104: // M = 13
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -281,15 +285,15 @@ LoopSzEnd_TILE_1:
.inst 0x05b022c7 // dup z7.q, z22.q[2]
.inst 0x05f022c8 // dup z8.q, z22.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -301,7 +305,7 @@ LoopSzEnd_TILE_1:
.inst 0xe400f5a7 // st1b {z7.b}, p5, [x13]
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
b TILE1_Dz_End
b End
TILE1_STORE96: // M = 12
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -314,14 +318,14 @@ LoopSzEnd_TILE_1:
.inst 0x05b022c7 // dup z7.q, z22.q[2]
.inst 0x05f022c8 // dup z8.q, z22.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -332,7 +336,7 @@ LoopSzEnd_TILE_1:
.inst 0xe4045566 // st1b {z6.b}, p5, [x11, x4]
.inst 0xe400f5a7 // st1b {z7.b}, p5, [x13]
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
b TILE1_Dz_End
b End
TILE1_STORE88: // M = 11
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -344,14 +348,14 @@ LoopSzEnd_TILE_1:
.inst 0x057022c6 // dup z6.q, z22.q[1]
.inst 0x05b022c7 // dup z7.q, z22.q[2]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -361,7 +365,7 @@ LoopSzEnd_TILE_1:
.inst 0xe400f576 // st1b {z22.b}, p5, [x11]
.inst 0xe4045566 // st1b {z6.b}, p5, [x11, x4]
.inst 0xe400f5a7 // st1b {z7.b}, p5, [x13]
b TILE1_Dz_End
b End
TILE1_STORE80: // M = 10
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -372,13 +376,13 @@ LoopSzEnd_TILE_1:
.inst 0x05f022a5 // dup z5.q, z21.q[3]
.inst 0x057022c6 // dup z6.q, z22.q[1]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -387,7 +391,7 @@ LoopSzEnd_TILE_1:
.inst 0xe4045505 // st1b {z5.b}, p5, [x8, x4]
.inst 0xe400f576 // st1b {z22.b}, p5, [x11]
.inst 0xe4045566 // st1b {z6.b}, p5, [x11, x4]
b TILE1_Dz_End
b End
TILE1_STORE72: // M = 9
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -397,13 +401,13 @@ LoopSzEnd_TILE_1:
.inst 0x05b022a4 // dup z4.q, z21.q[2]
.inst 0x05f022a5 // dup z5.q, z21.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -411,7 +415,7 @@ LoopSzEnd_TILE_1:
.inst 0xe400f504 // st1b {z4.b}, p5, [x8]
.inst 0xe4045505 // st1b {z5.b}, p5, [x8, x4]
.inst 0xe400f576 // st1b {z22.b}, p5, [x11]
b TILE1_Dz_End
b End
TILE1_STORE64: // M = 8
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -421,19 +425,19 @@ LoopSzEnd_TILE_1:
.inst 0x05b022a4 // dup z4.q, z21.q[2]
.inst 0x05f022a5 // dup z5.q, z21.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
.inst 0xe40455e3 // st1b {z3.b}, p5, [x15, x4]
.inst 0xe400f504 // st1b {z4.b}, p5, [x8]
.inst 0xe4045505 // st1b {z5.b}, p5, [x8, x4]
b TILE1_Dz_End
b End
TILE1_STORE56: // M = 7
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -442,18 +446,18 @@ LoopSzEnd_TILE_1:
.inst 0x057022a3 // dup z3.q, z21.q[1]
.inst 0x05b022a4 // dup z4.q, z21.q[2]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
.inst 0xe40455e3 // st1b {z3.b}, p5, [x15, x4]
.inst 0xe400f504 // st1b {z4.b}, p5, [x8]
b TILE1_Dz_End
b End
TILE1_STORE48: // M = 6
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -461,66 +465,66 @@ LoopSzEnd_TILE_1:
.inst 0x05f02282 // dup z2.q, z20.q[3]
.inst 0x057022a3 // dup z3.q, z21.q[1]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
.inst 0xe40455e3 // st1b {z3.b}, p5, [x15, x4]
b TILE1_Dz_End
b End
TILE1_STORE40: // M = 5
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0x05b02281 // dup z1.q, z20.q[2]
.inst 0x05f02282 // dup z2.q, z20.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
b TILE1_Dz_End
b End
TILE1_STORE32: // M = 4
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0x05b02281 // dup z1.q, z20.q[2]
.inst 0x05f02282 // dup z2.q, z20.q[3]
add x9, x6, x4, lsl #1 // +2
add x9, x0, x4, lsl #1 // +2
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
b TILE1_Dz_End
b End
TILE1_STORE24: // M = 3
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0x05b02281 // dup z1.q, z20.q[2]
add x9, x6, x4, lsl #1 // +2
add x9, x0, x4, lsl #1 // +2
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
b TILE1_Dz_End
b End
TILE1_STORE16: // M = 2
.inst 0x05702280 // dup z0.q, z20.q[1]
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
b TILE1_Dz_End
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
b End
TILE1_STORE8: // M = 1
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
b TILE1_Dz_End
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
b End
TILE1_STORE120:
@@ -536,16 +540,16 @@ LoopSzEnd_TILE_1:
.inst 0x057022e9 // dup z9.q, z23.q[1]
.inst 0x05b022ea // dup z10.q, z23.q[2]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
add x5, x8, x4, lsl #3 // +14
add x6, x8, x4, lsl #3 // +14
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -558,9 +562,9 @@ LoopSzEnd_TILE_1:
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
.inst 0xe40456e9 // st1b {z9.b}, p5, [x23, x4]
.inst 0xe400f4aa // st1b {z10.b}, p5, [x5]
.inst 0xe400f4ca // st1b {z10.b}, p5, [x6]
b TILE1_Dz_End
b End
TILE1_STORE128:
.inst 0x05702280 // dup z0.q, z20.q[1]
@@ -576,16 +580,16 @@ LoopSzEnd_TILE_1:
.inst 0x05b022ea // dup z10.q, z23.q[2]
.inst 0x05f022eb // dup z11.q, z23.q[3]
add x9, x6, x4, lsl #1 // +2
add x15, x6, x4, lsl #2 // +4
add x9, x0, x4, lsl #1 // +2
add x15, x0, x4, lsl #2 // +4
add x8, x9, x4, lsl #2 // +6
add x11, x6, x4, lsl #3 // +8
add x11, x0, x4, lsl #3 // +8
add x13, x9, x4, lsl #3 // +10
add x23, x15, x4, lsl #3 // +12
add x5, x8, x4, lsl #3 // +14
add x6, x8, x4, lsl #3 // +14
.inst 0xe400f4d4 // st1b {z20.b}, p5, [x6]
.inst 0xe40454c0 // st1b {z0.b}, p5, [x6, x4]
.inst 0xe400f414 // st1b {z20.b}, p5, [x0]
.inst 0xe4045400 // st1b {z0.b}, p5, [x0, x4]
.inst 0xe400f521 // st1b {z1.b}, p5, [x9]
.inst 0xe4045522 // st1b {z2.b}, p5, [x9, x4]
.inst 0xe400f5f5 // st1b {z21.b}, p5, [x15]
@@ -598,25 +602,22 @@ LoopSzEnd_TILE_1:
.inst 0xe40455a8 // st1b {z8.b}, p5, [x13, x4]
.inst 0xe400f6f7 // st1b {z23.b}, p5, [x23]
.inst 0xe40456e9 // st1b {z9.b}, p5, [x23, x4]
.inst 0xe400f4aa // st1b {z10.b}, p5, [x5]
.inst 0xe40454ab // st1b {z11.b}, p5, [x5, x4]
.inst 0xe400f4ca // st1b {z10.b}, p5, [x6]
.inst 0xe40454cb // st1b {z11.b}, p5, [x6, x4]
TILE1_Dz_End:
sub x24, x24, #16
cmp x24, #0
ble End
add x6, x6, x4, LSL #4
subs x5, x5, #16
add x0, x0, x4, LSL #4
beq End
mov x13, x19
mov x23, x21
cbz x27, LoopDz_TILE1
REVERT_INPUT_DEQUANT_BIAS x27, x15, x26, x25
mov x27, x24
b LoopDz_TILE1
End:
.inst 0xd503467f // smstop
ldp x19, x20, [sp, #224]
ldp x21, x22, [sp, #208]
ldp x23, x24, [sp, #192]
@@ -628,5 +629,5 @@ ldp d12, d13, [sp, #48]
ldp d14, d15, [sp, #32]
ldp x29, x30, [sp], #320
ret
#endif // __aarch64__
#endif // __aarch64__
source/backend/cpu/arm/arm64/sme2_asm/MNNGemmInt8AddBiasScaleHp128_SME2_w8_Fp32.S
+357 -360
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/arm/arm64/sme2_asm/MNNPackedMatMulRemainFP32_SME2.S
+191 -1811
View File
File diff suppressed because it is too large Load Diff
source/backend/cpu/compute/CommonOptFunction.cpp
+1 -2
View File
@@ -4135,8 +4135,7 @@ static void Sme2MNNPackForMatMul_B(float* destC, const float* sourceC, size_t h,
}
}
static void Sme2MNNPackC4ForMatMul_A(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el) {
const int32_t infosme2[4] = {info[0], info[1], SME2_MATMUL_EP, info[3]};
MNNPackC4ForMatMul_A(destOrigin, sourceGroup, infosme2, el);
MNNPackC4ForMatMul_A(destOrigin, sourceGroup, info, el);
return;
}
#endif
source/backend/cpu/compute/ConvInt8TiledExecutor.cpp
+12 -6
View File
@@ -336,7 +336,7 @@ DenseConvInt8TiledExecutor::DenseConvInt8TiledExecutor(Backend* backend, const O
auto quantlen = 2 * blockNum * ROUND_UP(oc, UNIT) * QUANT_INFO_BYTES;
auto weightlen = shape[0] * shape[1] * shape[2] * shape[3] * shape[4];
mResourceInt8->mWeightInt8.reset(Tensor::createDevice<uint8_t>({weightlen + quantlen}));
mResourceInt8->mOriginBias.reset(Tensor::createDevice<int32_t>({ocUpHp})); // float
mResourceInt8->mOriginBias.reset(Tensor::createDevice<int32_t>({ocUp4})); // float
if (inputBlockQuantOption != 2) {
mResourceInt8->mWeightKernelSum.reset(Tensor::createDevice<uint8_t>({QUANT_INFO_BYTES * ocUpHp}));
} else {
@@ -357,7 +357,7 @@ DenseConvInt8TiledExecutor::DenseConvInt8TiledExecutor(Backend* backend, const O
}
// read weight, weight's scale&bias, convolution bias
::memset(mResourceInt8->mOriginBias->host<float>(), 0, ocUpHp * sizeof(float));
::memset(mResourceInt8->mOriginBias->host<float>(), 0, ocUp4 * sizeof(float));
// dynamic quant
bool directReadInt4weight = (kernelCount == 1 && ROUND_UP(oc, UNIT) == oc && ROUND_UP(ic, SRC_UNIT) == ic);
@@ -506,7 +506,11 @@ DenseConvInt8TiledExecutor::DenseConvInt8TiledExecutor(Backend* backend, const O
}
if (false == weightAsy) { // symmetric quant
::memcpy(scaleAndBias.get(), convOp->quanParameter()->alpha()->data(), quantCount * sizeof(float));
if (convOp->quanParameter() && convOp->quanParameter()->alpha()) {
::memcpy(scaleAndBias.get(), convOp->quanParameter()->alpha()->data(), quantCount * sizeof(float));
} else {
::memcpy(scaleAndBias.get(), quanCommon->alpha.get(), quanCommon->alpha.size() * sizeof(float));
}
} else if (true == weightAsy) { // asymmetric
int scaleSize = quantCount / 2;
for (int i = 0; i < scaleSize; ++i) {
@@ -658,7 +662,7 @@ ErrorCode DenseConvInt8TiledExecutor::onResize(const std::vector<Tensor*>& input
mTileCount = UP_DIV(planeSize, DynamicDestUnit);
if (mTileCount > threads || (mOnlineReorderWeightSme && planeSize > 1)) {
mSplitByOc = false;
}
}
}
if (mSplitByOc) {
@@ -717,7 +721,10 @@ ErrorCode DenseConvInt8TiledExecutor::onResize(const std::vector<Tensor*>& input
mTempSrcSum = bufferAlloc->alloc(mTileCount * mBlockNum * DST_XUNIT * mIm2ColCount * QUANT_INFO_BYTES);
}
mAccumBuffer.reset(Tensor::createDevice<int32_t>({threads, DST_XUNIT * ALIMAX(UNIT, gcore->pack)}));
auto success = backend()->onAcquireBuffer(mTempIm2ColBuffer.get(), Backend::DYNAMIC);
success &= backend()->onAcquireBuffer(mAccumBuffer.get(), Backend::DYNAMIC);
if (!success || mBlitInfo.invalid() || mTempSrcSum.invalid()) {
return OUT_OF_MEMORY;
}
@@ -728,6 +735,7 @@ ErrorCode DenseConvInt8TiledExecutor::onResize(const std::vector<Tensor*>& input
if (mBatchQuantInfo.get()) {
backend()->onReleaseBuffer(mBatchQuantInfo.get(), Backend::DYNAMIC);
}
backend()->onReleaseBuffer(mAccumBuffer.get(), Backend::DYNAMIC);
return NO_ERROR;
}
@@ -853,8 +861,6 @@ ErrorCode DenseConvInt8TiledExecutor::onResize(const std::vector<Tensor*>& input
return OUT_OF_MEMORY;
}
}
mAccumBuffer.reset(Tensor::createDevice<int32_t>({threads, DST_XUNIT * ALIMAX(UNIT, gcore->pack)}));
success &= backend()->onAcquireBuffer(mAccumBuffer.get(), Backend::DYNAMIC);
if (mBlockNum > 1 && kernelCount > 1) {
if (mSplitByOc) {
source/backend/cpu/compute/ConvolutionPackFreeWinograd.cpp
+1 -1
View File
@@ -113,7 +113,7 @@ ErrorCode ConvolutionPackFreeWinograd::onExecute(const std::vector<Tensor *> &in
int tileCount = UP_DIV(totalCount, mConvPerfconfig.eTile);
std::vector<size_t> parameters(7);
parameters[0] = eRemain * bytes;
parameters[0] = eRemain * lPack * bytes;
parameters[1] = ROUND_UP(input->channel(), lPack);
parameters[2] = output->channel();
parameters[3] = ePack * pack * bytes;
source/backend/cpu/compute/ConvolutionPackWinograd.cpp
+5 -4
View File
@@ -260,7 +260,7 @@ ErrorCode ConvolutionPackWinograd::onResize(const std::vector<Tensor *> &inputs,
auto totalCount = wUnit * hUnit * batch;
// MNN_PRINT("ow=%d, oh=%d\n", ow, oh);
std::vector<int> divides(threadNumber+1);
static_cast<CPUBackend *>(backend())->computeDivideSizes(totalCount, divides.data()+1);
divides[0] = 0;
@@ -281,15 +281,16 @@ ErrorCode ConvolutionPackWinograd::onResize(const std::vector<Tensor *> &inputs,
}
int eRemain = (tFin-tSta) % ePack;
std::vector<size_t> parameters(6);
parameters[0] = ePack * lPack * bytes;
parameters[1] = ROUND_UP(input->channel(), lPack);
parameters[2] = output->channel();
parameters[3] = ePack * pack * bytes;
parameters[4] = 0;
parameters[5] = 0;
parameters[0] = eRemain * bytes;
parameters[3] = ePack * pack * bytes;
std::vector<size_t> parametersRemain = parameters;
parametersRemain[0] = eRemain * bytes;
parametersRemain[0] = eRemain * lPack * bytes;
parametersRemain[3] = eRemain * pack * bytes;
auto srcOrigin = inputOrigin;
source/backend/cpu/compute/DenseConvolutionTiledExecutor.cpp
+8 -16
View File
@@ -40,7 +40,7 @@ bool DenseConvolutionTiledExecutor::initQuantizeResource(std::shared_ptr<Convolu
int scaleSize = dequantCnt; // real size
if (int8Info->asymmetric) {
scaleSize = dequantCnt / 2;
}
int blockNum = scaleSize / outputCount;
scaleSize = blockNum * hU * hP; // pack size
@@ -304,10 +304,10 @@ ErrorCode ConvolutionTiledExecutorMultiInput::onResize(const std::vector<Tensor*
if (!res) {
return OUT_OF_MEMORY;
}
if (inputs.size() > 2 && inputs[2]->elementSize() % hP == 0) {
if (inputs.size() > 2 && inputs[2]->elementSize() % function->pack == 0) {
mInputs = {inputs[0], mTempWeight.get(), inputs[2]};
} else {
auto hPackedSize = ALIMAX(hP, function->pack);
auto hPackedSize = function->pack;
mTempBias.reset(Tensor::createDevice<float>({UP_DIV(outputCount, hPackedSize) * hPackedSize}));
backend()->onAcquireBuffer(mTempBias.get(), Backend::DYNAMIC);
mInputs = {inputs[0], mTempWeight.get(), mTempBias.get()};
@@ -469,7 +469,7 @@ ErrorCode DenseConvolutionTiledImpl::onResize(const std::vector<Tensor*>& inputs
auto output = outputs[0];
auto batch = output->batch();
int threadNumber = ((CPUBackend *)backend())->threadNumber();
int LRoundup = ROUND_UP(L, lP);
int LRoundupC4 = UP_DIV(LRoundup, unit);
auto outputChannel = output->channel();
@@ -523,7 +523,7 @@ ErrorCode DenseConvolutionTiledImpl::onResize(const std::vector<Tensor*>& inputs
info[3] = mIm2ColParameters.strideX;
size_t shapeParameters[PARAMETERSIZE];
size_t* parameters = shapeParameters;
parameters[0] = eP * bytes;
parameters[0] = eP * lP * bytes;
parameters[1] = blockSize;
parameters[2] = outputChannel;
parameters[3] = plane * unit * bytes;
@@ -651,7 +651,7 @@ ErrorCode DenseConvolutionTiledImpl::onResize(const std::vector<Tensor*>& inputs
info[2] = eP;
info[3] = mIm2ColParameters.strideX;
size_t parameters[PARAMETERSIZE];
parameters[0] = eP * bytes;
parameters[0] = eP * lP * bytes;
parameters[1] = blockSize;
parameters[2] = outputChannel;
parameters[3] = plane * unit * bytes;
@@ -678,14 +678,6 @@ ErrorCode DenseConvolutionTiledImpl::onResize(const std::vector<Tensor*>& inputs
if (number > 0) {
packA((float *)gemmBuffer, srcPtr, info, el);
}
/*
for (int kk=0; kk < mIm2ColParameters.kernelX * mIm2ColParameters.kernelY; ++kk) {
for (int xx=0; xx < ROUND_UP(input->channel(), lP) * eP; ++xx) {
printf("%f ", ((__fp16*)gemmBuffer)[kk * ROUND_UP(input->channel(), lP) * eP + xx]);
if (xx % (eP * lP) == (eP * lP -1)) printf("\n");
}
}
*/
int finishedL = 0;
int wquantStride = 0;
int8_t* _weightPtr = reinterpret_cast<int8_t*>(weightPtr);
@@ -702,7 +694,7 @@ ErrorCode DenseConvolutionTiledImpl::onResize(const std::vector<Tensor*>& inputs
}
finishedL = blockSize * bk;
wquantStride = static_cast<int32_t>(blockSize * bk * hP * halfStride);
matmulUnit(_dstFloatPtr, (float*)(gemmBuffer + bytes * eP * finishedL), (float*)(_weightPtr + wquantStride), parameters, relufp32, exeBiasPtr, (float*)(dequantAlpha + bk * ocUp4 * bytes), (float*)(dequantBias + bk * ocUp4 * bytes));
}
} else {
@@ -714,7 +706,7 @@ ErrorCode DenseConvolutionTiledImpl::onResize(const std::vector<Tensor*>& inputs
}
finishedL = blockSize * bk;
wquantStride = static_cast<int32_t>(blockSize * bk * hP * halfStride);
matmulRemain(_dstFloatPtr, (float*)(gemmBuffer + eP * bytes * finishedL), (float*)(_weightPtr + wquantStride), xC, parameters, relufp32, exeBiasPtr, (float*)(dequantAlpha + bk * ocUp4 * bytes), (float*)(dequantBias + bk * ocUp4 * bytes ));
}
// matmulRemain(_dstFloatPtr, (float*)gemmBuffer, (float*)weightPtr, xC, parameters, postParameters.data(), biasPtr, k, b);
source/backend/cpu/compute/StrassenMatmulComputor.cpp
+5 -5
View File
@@ -80,7 +80,7 @@ ErrorCode StrassenMatrixComputor::_generateTrivalMatMul(int e, int l, int h, con
std::make_pair([cStride, l, h, xCount, AT, BT, CT, COT, tileBufferBasic, unitNumber, bExtraStride, numberThread, eReal, eP, lP, active, matmulUnit, matmulRemain, this](int tId) {
auto core = static_cast<CPUBackend*>(backend())->functions();
size_t parameters[7];
parameters[0] = xCount * core->bytes;
parameters[0] = xCount * lP * core->bytes;
parameters[1] = ROUND_UP(l, lP);
parameters[2] = h;
parameters[3] = cStride;
@@ -305,7 +305,7 @@ ErrorCode StrassenMatrixComputor::_generateMatMul(int e, int l, int h, const Mat
CX.stackIndex = X.stackIndex;
CX.offsetBytes = 0;
CX.lineStrideBytes = eSub * core->bytes * core->pack;
MatrixInfo a11 = AT;
MatrixInfo a12 = AT;
a12.offsetBytes = AT.offsetBytes + AT.lineStrideBytes * lSubUnit;
@@ -313,7 +313,7 @@ ErrorCode StrassenMatrixComputor::_generateMatMul(int e, int l, int h, const Mat
a21.offsetBytes = AT.offsetBytes + eSub * core->pack * core->bytes;
MatrixInfo a22 = AT;
a22.offsetBytes = AT.offsetBytes + eSub * core->pack * core->bytes + AT.lineStrideBytes * lSubUnit;
MatrixInfo b11 = BT;
MatrixInfo b12 = BT;
b12.offsetBytes = BT.offsetBytes + BT.lineStrideBytes * (hSub / hP);
@@ -321,7 +321,7 @@ ErrorCode StrassenMatrixComputor::_generateMatMul(int e, int l, int h, const Mat
b21.offsetBytes = BT.offsetBytes + lSub * hP * mWeightBytes;
MatrixInfo b22 = BT;
b22.offsetBytes = BT.offsetBytes + BT.lineStrideBytes * (hSub / hP) + lSub * hP * mWeightBytes;
MatrixInfo c11 = CT;
MatrixInfo c12 = CT;
c12.offsetBytes = CT.offsetBytes + CT.lineStrideBytes * (hSub / core->pack);
@@ -533,7 +533,7 @@ ErrorCode StrassenMatrixComputor::onEncode(int e, int l, int h, int as, int bs,
b.stackIndex = 1;
b.lineStrideBytes = bs * mWeightBytes;
b.offsetBytes = 0;
c.stackIndex = 2;
c.lineStrideBytes = cs * core->bytes;
c.offsetBytes = 0;
source/backend/cpu/x86_x64/AVX2Backend.cpp
+3
View File
@@ -363,6 +363,9 @@ void AVX2Backend::onCopyBuffer(const Tensor* srcTensor, const Tensor* dstTensor)
return;
}
_resetDynamicMemory();
if (mRuntime->pCurrentStatus != NO_ERROR) {
return;
}
if (getDataType(srcTensor) != getDataType(dstTensor)) {
auto dimType = Tensor::CAFFE;
switch (TensorUtils::getDescribe(srcTensor)->dimensionFormat) {
source/backend/metal/AllShader.cpp
+1 -72
View File
@@ -466,77 +466,6 @@ const char* shader_MetalConvolution_metal =
" if (valids) { z_out += cst.output_size*cst.batch; *z_out=activate(M4(result1+FLOAT4(biasTerms[uz[1]])),cst.activation); }\n"
"}\n"
;
const char* shader_MetalReduction_metal =
"struct reduce_shape {\n"
" int outside_size;\n"
" int axis_size;\n"
" int inside_size;\n"
" int outside_step;\n"
"};\n"
"template <typename M,typename T>\n"
"static inline void reduce_mean(const device T *in,device T *out,constant reduce_shape &s,uint2 gid) {\n"
" auto axis_in=in+gid.x*s.outside_step+gid.y;\n"
" M summer=0;\n"
" for (int i=0; i<s.axis_size; i++,axis_in += s.inside_size) {\n"
" summer += M(*axis_in);\n"
" }\n"
" out[int(gid.x)*s.inside_size+int(gid.y)]=T(summer/s.axis_size);\n"
"}\n"
"template <typename M,typename T>\n"
"static inline void reduce_sum(const device T *in,device T *out,constant reduce_shape &s,uint2 gid) {\n"
" auto axis_in=in+gid.x*s.outside_step+gid.y;\n"
" M summer=0;\n"
" for (int i=0; i<s.axis_size; i++,axis_in += s.inside_size) {\n"
" summer += M(*axis_in);\n"
" }\n"
" out[int(gid.x)*s.inside_size+int(gid.y)]=T(summer);\n"
"}\n"
"template <typename M,typename T>\n"
"static inline void reduce_min(const device T *in,device T *out,constant reduce_shape &s,uint2 gid) {\n"
" auto axis_in=in+gid.x*s.outside_step+gid.y;\n"
" T summer=*axis_in; axis_in += s.inside_size;\n"
" for (int i=1; i<s.axis_size; i++,axis_in += s.inside_size) {\n"
" summer=min(summer,*axis_in);\n"
" }\n"
" out[int(gid.x)*s.inside_size+int(gid.y)]=summer;\n"
"}\n"
"template <typename M,typename T>\n"
"static inline void reduce_max(const device T *in,device T *out,constant reduce_shape &s,uint2 gid) {\n"
" auto axis_in=in+gid.x*s.outside_step+gid.y;\n"
" T summer=*axis_in; axis_in += s.inside_size;\n"
" for (int i=1; i<s.axis_size; i++,axis_in += s.inside_size) {\n"
" summer=max(summer,*axis_in);\n"
" }\n"
" out[int(gid.x)*s.inside_size+int(gid.y)]=summer;\n"
"}\n"
"template <typename M,typename T>\n"
"static inline void reduce_prod(const device T *in,device T *out,constant reduce_shape &s,uint2 gid) {\n"
" auto axis_in=in+gid.x*s.outside_step+gid.y;\n"
" M summer=1;\n"
" for (int i=0; i<s.axis_size; i++,axis_in += s.inside_size) {\n"
" summer *= M(*axis_in);\n"
" }\n"
" out[int(gid.x)*s.inside_size+int(gid.y)]=T(summer);\n"
"}\n"
"#define define_reduce(name) "
"kernel void reduce_##name##_f(const device M *in [[buffer(0)]],"
" device M *out [[buffer(1)]],"
" constant reduce_shape &s [[buffer(2)]],"
" uint2 gid [[thread_position_in_grid]]) { "
" if (gid.x<(uint)s.outside_size && gid.y<(uint)s.inside_size) reduce_##name<FLOAT,M>(in,out,s,gid); "
"} "
"kernel void reduce_##name##_s(const device int *in [[buffer(0)]],"
" device int *out [[buffer(1)]],"
" constant reduce_shape &s [[buffer(2)]],"
" uint2 gid [[thread_position_in_grid]]) { "
" if (gid.x<(uint)s.outside_size && gid.y<(uint)s.inside_size) reduce_##name<int,int>(in,out,s,gid); "
"}\n"
"define_reduce(mean);\n"
"define_reduce(sum);\n"
"define_reduce(min);\n"
"define_reduce(max);\n"
"define_reduce(prod);\n"
;
const char* shader_MetalSoftmax_metal =
"struct softmax_shape {\n"
" int inside_size;\n"
@@ -564,7 +493,7 @@ const char* shader_MetalSoftmax_metal =
" auto axis_out=out+axis_off;\n"
" \n"
" // get max\n"
" float max1=-INFINITY;\n"
" float max1=-FLT_MAX;\n"
" for (int i=0; i<s.axis_length; i++) {\n"
" max1=max(max1,float(axis_in[i*s.inside_size]));\n"
" }\n"
source/backend/metal/AllShader.hpp
-1
View File
@@ -4,7 +4,6 @@ extern const char* shader_MetalReLU6_metal;
extern const char* shader_MetalConvolutionDepthwise_metal;
extern const char* shader_MetalConvolutionActivation_metal;
extern const char* shader_MetalConvolution_metal;
extern const char* shader_MetalReduction_metal;
extern const char* shader_MetalSoftmax_metal;
extern const char* shader_MetalLayerNorm_metal;
extern const char* shader_MetalConvolutionWinograd_metal;
source/backend/metal/ConvSimdGroupShader.hpp
+377 -1
View File
@@ -18,6 +18,16 @@ typedef enum : int {
ReLU6 = 2,
} conv_activation_type;
inline ftype2 activate(ftype2 value, conv_activation_type type) {
switch (type) {
case ReLU:
return max(value, (ftype2)0);
case ReLU6:
return clamp(value, (ftype2)0, (ftype2)6);
default: // None
return value;
}
}
inline ftype4 activate(ftype4 value, conv_activation_type type) {
switch (type) {
case ReLU:
@@ -80,6 +90,372 @@ typedef half4x4 FLOAT4x4;
simdgroup_store(sgd[i], ptr + 64*i, 8);\
}
kernel void conv1x1_gemm_8x8_wquant_sg(const device ftype2 *in [[buffer(0)]],
device ftype2 *out [[buffer(1)]],
constant conv1x1_constants& cst [[buffer(2)]],
#ifdef W_QUANT_4
const device uchar *wt [[buffer(3)]],
#elif defined(W_QUANT_8)
const device char2 *wt [[buffer(3)]],
#endif
const device ftype2 *biasTerms [[buffer(4)]],
const device ftype2 *dequantScale [[buffer(5)]],
uint3 gid [[threadgroup_position_in_grid]],
uint tiitg[[thread_index_in_threadgroup]],
uint sgitg[[simdgroup_index_in_threadgroup]]) {
/*
// Read:
ftype 0~63 ---> input: [M8, K8]
ftype 64~127 ---> input: [K8, N8]
// Write:
ftype 0~63 ---> input: [M8, N8]
*/
threadgroup FLOAT sdata[128] = {0.f};
INIT_SIMDGROUP_MATRIX(1, 1, 1);
int rx = gid.x;// M/8
int uz = gid.y;// N/8
int kl = tiitg / 16; // 0~1
int rcl = tiitg % 16; // 0~15
int kr = rcl % 2; // 0~1
int ml = rcl / 2; // 0 ~ 7
int nl = ml / 2; // 0 ~ 3
int nr = ml % 2; // 0 ~ 1
/** input:
threadgroup: [M8, K8]
each thread: K2
layout: [K/4, M, K4] -> [K/8, K2, M/8, M8, K2, K2]
index : [0, kr, rx, ml, kl, K2]
offset: ((0*2+kr) * M + rx * 8 + ml) * 2 + kl
*/
/** weight:
threadgroup: [K8, N8]
each thread: K2
layout: [N/4, K/4, N4, K2, K2] -> [N/8, N2, K/8, K2, N4, K2, K2]
index : [uz, nr, 0, kr, nl, kl, K2]
offset: (((uz * 2 + nr) * K/4 + 0*2+kr) * 4 + nl) * 2 + kl
*/
/** output:
threadgroup: [M8, N8] -> [M8, N4, N2]
sdata: [ml, kr * 2 + kl]
each thread: N4
layout: [N/4, M, N4] -> [N/8, N2, M/8, M8, N2, N2]
index : [uz, kr, rx, ml, kl, N2]
offset: (((uz * 2 + kr) * M + rx * 8 + ml) * 2 + kl)
*/
// boundary limit
int idx_n4 = (uz * 2 + nr) < cst.output_slice ? (uz * 2 + nr) : (cst.output_slice - 1);
int idx_m = (rx * 8 + ml) < cst.input_size * cst.batch ? (rx * 8 + ml) : (cst.input_size * cst.batch - 1);
auto xy_wt = wt + ((idx_n4 * cst.input_slice + 0*2+kr) * 4 + nl) * 2 + kl;// [N/4, K/4, N4, K4]
auto xy_in0 = in + ((0*2+kr) * cst.input_size * cst.batch + idx_m) * 2 + kl;// [K/4, M, K2, K2]
auto xy_out = out + ((uz * 2 + kr) * cst.output_size * cst.batch + rx * 8 + ml) * 2 + kl;// [N/4, M, N4]
int block = (cst.input_slice + cst.block_size - 1) / cst.block_size;
for (int bi=0; bi<cst.block_size; ++bi) {
// [N/4, cst.block_size, 2/*scale_bias*/, N2， N2]
FLOAT2 scale = FLOAT2(dequantScale[(2 * (idx_n4 * cst.block_size + bi) + 0) * 2 + nl / 2]) / (FLOAT)cst.scale_coef;
FLOAT2 dequant_bias = FLOAT2(dequantScale[(2 * (idx_n4 * cst.block_size + bi) + 1) * 2 + nl / 2]) / (FLOAT)cst.scale_coef;
int zmin = bi * block;
int zmax = min(zmin + block, cst.input_slice);
for (int z = zmin; z < zmax; z += 2) {
// [M8, K2, K2, K2]
((threadgroup FLOAT2*)sdata)[(ml * 2 + kr) * 2 + kl] = (FLOAT2)(*xy_in0);
xy_in0 += 4 * cst.input_size * cst.batch;
#ifdef W_QUANT_4
uchar w_int40 = xy_wt[8 * z]; // [N/4, K/4, N4, K4]
FLOAT2 w20 = FLOAT2((float)(w_int40 >> 4) - 8, (float)(w_int40 & 15) - 8);
#elif defined(W_QUANT_8)
char2 w_int40 = xy_wt[8 * z]; // [N/4, K/4, N4, K4]
FLOAT2 w20 = FLOAT2((float)w_int40[0], (float)w_int40[1]);
#endif
FLOAT2 res = w20 * scale[nl % 2] + dequant_bias[nl % 2];
// [K8, N4, N2]
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + kl * 2 + 0) * 8 + nr * 4 + nl] = res[0];
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + kl * 2 + 1) * 8 + nr * 4 + nl] = res[1];
threadgroup_barrier(mem_flags::mem_threadgroup);
simdgroup_load(sga[0], (const threadgroup FLOAT*)sdata, 8);
simdgroup_load(sgb[0], ((const threadgroup FLOAT*)sdata) + 64, 8);
SIMDGROUP_MATRIX_FMA(1, 1);
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
SIMDGROUP_MATRIX_STORE((threadgroup FLOAT*)sdata, 1);
threadgroup_barrier(mem_flags::mem_threadgroup);
if((rx * 8 + ml) < cst.input_size * cst.batch) {
if((uz * 2 + kr) < cst.output_slice) {
xy_out[0] = activate(ftype2(((threadgroup FLOAT2*)sdata)[ml * 4 + kr * 2 + kl] + FLOAT2(biasTerms[(uz * 2 + kr) * 2 + kl])), cst.activation);
}
}
}
kernel void conv1x1_gemm_8x16_wquant_sg(const device ftype2 *in [[buffer(0)]],
device ftype4 *out [[buffer(1)]],
constant conv1x1_constants& cst [[buffer(2)]],
#ifdef W_QUANT_4
const device uchar2 *wt [[buffer(3)]],
#elif defined(W_QUANT_8)
const device char4 *wt [[buffer(3)]],
#endif
const device ftype4 *biasTerms [[buffer(4)]],
const device ftype4 *dequantScale [[buffer(5)]],
uint3 gid [[threadgroup_position_in_grid]],
uint tiitg[[thread_index_in_threadgroup]],
uint sgitg[[simdgroup_index_in_threadgroup]]) {
/*
// Read:
ftype 0~63 ---> input: [M8, K8]
ftype 64~191 ---> input: [K8, N16]
// Write:
ftype 0~127 ---> input: [N2, M8, N8]
*/
threadgroup FLOAT sdata[256] = {0.f};
INIT_SIMDGROUP_MATRIX(1, 2, 2);
int rx = gid.x;// M/8
int uz = gid.y;// N/16
int kl = tiitg / 16; // 0~1
int rcl = tiitg % 16; // 0~15
/** input:
threadgroup: [M8, K8]
each thread: K2
layout: [K/4, M, K4] -> [K/8, K2, M/8, M8, K2, K2]
index : [0, rcl/8, rx, rcl%8, kl, K2]
offset: ((0*2+rcl/8) * M + rx * 8 + rcl%8) * 2 + kl
*/
/** weight:
threadgroup: [K8, N16]
each thread: K4
layout: [N/4, K/4, N4, K4] -> [N/16, N4, K/8, K2, N4, K4]
index : [uz, rcl/4, 0, kl, rcl%4, K4]
offset: (((uz * 4 + rcl/4) * K/4 + 0*2+kl) * 4 + rcl%4)
*/
/** output:
threadgroup: [M8, N16] -> [N2, M8, N2, N4]
sdata: [(rcl / 4) / 2, (rcl%4) * 2 + kl, (rcl / 4) % 2]
each thread: N4
layout: [N/4, M, N4] -> [N/16, N4, M/8, M4, M2, N4]
index : [uz, rcl/4, rx, rcl%4, kl, N4]
offset: ((uz * 4 + rcl/4) * M + (rx * 8 + (rcl%4) * 2 + kl))
*/
// boundary limit
int idx_n4 = (4 * uz + rcl / 4) < cst.output_slice ? (4 * uz + rcl / 4) : (cst.output_slice - 1);
int idx_m = (8 * rx + rcl%8) < cst.input_size * cst.batch ? (8 * rx + rcl%8) : (cst.input_size * cst.batch - 1);
auto xy_wt = wt + ((idx_n4 * cst.input_slice + 0*2+kl) * 4 + rcl % 4);// [N/4, K/4, N4, K4]
auto xy_in0 = in + ((0*2+rcl/8) * cst.input_size * cst.batch + idx_m) * 2 + kl;// [K/4, M, K2, K2]
auto xy_out = out + (4 * uz + rcl / 4) * cst.output_size * cst.batch + (rx * 8 + (rcl%4) * 2 + kl);// [N/4, M, N4]
int block = (cst.input_slice + cst.block_size - 1) / cst.block_size;
for (int bi=0; bi<cst.block_size; ++bi) {
// [N/4, cst.block_size, 2/*scale_bias*/, N4]
FLOAT4 scale = FLOAT4(dequantScale[2 * (idx_n4 * cst.block_size + bi) + 0]) / (FLOAT)cst.scale_coef;
FLOAT4 dequant_bias = FLOAT4(dequantScale[2 * (idx_n4 * cst.block_size + bi) + 1]) / (FLOAT)cst.scale_coef;
int zmin = bi * block;
int zmax = min(zmin + block, cst.input_slice);
for (int z = zmin; z < zmax; z += 2) {
// [M8, K2, K2, K2]
((threadgroup FLOAT2*)sdata)[((rcl%8) * 2 + (rcl/8)) * 2 + kl] = (FLOAT2)(*xy_in0);
xy_in0 += 4 * cst.input_size * cst.batch;
#ifdef W_QUANT_4
uchar2 w_int40 = xy_wt[4 * z]; // [N/4, K/4, N4, K4]
FLOAT4 w40 = FLOAT4((float)(w_int40[0] >> 4) - 8, (float)(w_int40[0] & 15) - 8, (float)(w_int40[1] >> 4) - 8, (float)(w_int40[1] & 15) - 8);
#elif defined(W_QUANT_8)
char4 w_int40 = xy_wt[4 * z]; // [N/4, K/4, N4, K4]
FLOAT4 w40 = FLOAT4((float)w_int40[0], (float)w_int40[1], (float)w_int40[2], (float)w_int40[3]);
#endif
FLOAT4 res = w40 * scale[rcl % 4] + dequant_bias[rcl % 4];
// [K8, N4, N4]
((threadgroup FLOAT*)sdata)[64 + (kl * 4 + 0) * 16 + rcl] = res[0];
((threadgroup FLOAT*)sdata)[64 + (kl * 4 + 1) * 16 + rcl] = res[1];
((threadgroup FLOAT*)sdata)[64 + (kl * 4 + 2) * 16 + rcl] = res[2];
((threadgroup FLOAT*)sdata)[64 + (kl * 4 + 3) * 16 + rcl] = res[3];
threadgroup_barrier(mem_flags::mem_threadgroup);
simdgroup_load(sga[0], (const threadgroup FLOAT*)sdata, 8);
simdgroup_load(sgb[0], ((const threadgroup FLOAT*)sdata) + 64, 16);
simdgroup_load(sgb[1], ((const threadgroup FLOAT*)sdata) + 72, 16);
SIMDGROUP_MATRIX_FMA(1, 2);
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
SIMDGROUP_MATRIX_STORE((threadgroup FLOAT*)sdata, 2);
threadgroup_barrier(mem_flags::mem_threadgroup);
if((rx * 8 + (rcl%4) * 2 + kl) < cst.input_size * cst.batch) {
if((4 * uz + rcl / 4) < cst.output_slice) {
xy_out[0] = activate(ftype4(((threadgroup FLOAT4*)sdata)[(((rcl / 4) / 2) * 8 + ((rcl%4) * 2 + kl)) * 2 + (rcl / 4) % 2] + FLOAT4(biasTerms[4 * uz + rcl / 4])), cst.activation);
}
}
}
kernel void conv1x1_gemm_8x32_wquant_sg(const device ftype2 *in [[buffer(0)]],
device ftype4 *out [[buffer(1)]],
constant conv1x1_constants& cst [[buffer(2)]],
#ifdef W_QUANT_4
const device uchar2 *wt [[buffer(3)]],
#elif defined(W_QUANT_8)
const device char4 *wt [[buffer(3)]],
#endif
const device ftype4 *biasTerms [[buffer(4)]],
const device ftype4 *dequantScale [[buffer(5)]],
uint3 gid [[threadgroup_position_in_grid]],
uint tiitg[[thread_index_in_threadgroup]],
uint sgitg[[simdgroup_index_in_threadgroup]]) {
/*
// Read:
ftype 0~63 ---> input: [M8, K8]
ftype 64~319 ---> input: [K8, N32]
// Write:
ftype 0~255 ---> input: [N4, M8, N8]
*/
threadgroup FLOAT sdata[512] = {0.f};
INIT_SIMDGROUP_MATRIX(1, 4, 4);
int rx = gid.x;// M/8
int uz = gid.y;// N/32
int kl = tiitg / 16; // 0~1
int rcl = tiitg % 16; // 0~15
int kr = rcl % 2; // 0~1
int ml = rcl / 2; // 0 ~ 7
/** input:
threadgroup: [M8, K8]
each thread: K2
layout: [K/4, M, K4] -> [K/8, K2, M/8, M8, K2, K2]
index : [0, kr, rx, ml, kl, K2]
offset: ((0*2+kr) * M + rx * 8 + ml) * 2 + kl
*/
/** weight:
threadgroup: [K8, N32]
each thread: N2K4
layout: [N/4, K/4, N4, K4] -> [N/32, N8, K/8, K2, N2, N2, K4]
index : [uz, ml, 0, kr, kl, N2, K4]
offset: (((uz * 8 + ml) * K/4 + 0*2+kr) * 4 + kl * 2)
*/
/** output:
threadgroup: [M8, N32] -> [N4, M4, M2, N2, N4]
sdata: [ml/2, kr*2+kl, M2, ml%2, N4]
each thread: M2N4
layout: [N/4, M, N4] -> [N/32, N8, M/8, M4, M2, N4]
index : [uz, ml, rx, kr*2+kl, M2, N4]
offset: ((uz * 8 + ml) * M + (rx * 8 + (kr*2+kl) * 2 + 0/1))
*/
// boundary limit
int idx_n4 = (uz * 8 + ml) < cst.output_slice ? (uz * 8 + ml) : (cst.output_slice - 1);
int idx_m = (rx * 8 + ml) < cst.input_size * cst.batch ? (rx * 8 + ml) : (cst.input_size * cst.batch - 1);
auto xy_wt = wt + ((idx_n4 * cst.input_slice + 0*2+kr) * 4 + kl * 2);// [N/4, K/4, N4, K4]
auto xy_in0 = in + ((0*2+kr) * cst.input_size * cst.batch + idx_m) * 2 + kl;// [K/4, M, K2, K2]
auto xy_out = out + (uz * 8 + ml) * cst.output_size * cst.batch + (rx * 8 + (kr*2+kl) * 2);// [N/4, M, N4]
int block = (cst.input_slice + cst.block_size - 1) / cst.block_size;
for (int bi=0; bi<cst.block_size; ++bi) {
// [N/4, cst.block_size, 2/*scale_bias*/, N4]
FLOAT4 scale = FLOAT4(dequantScale[2 * (idx_n4 * cst.block_size + bi) + 0]) / (FLOAT)cst.scale_coef;
FLOAT4 dequant_bias = FLOAT4(dequantScale[2 * (idx_n4 * cst.block_size + bi) + 1]) / (FLOAT)cst.scale_coef;
int zmin = bi * block;
int zmax = min(zmin + block, cst.input_slice);
for (int z = zmin; z < zmax; z += 2) {
// [M8, K2, K2, K2]
((threadgroup FLOAT2*)sdata)[(ml * 2 + kr) * 2 + kl] = (FLOAT2)(*xy_in0);
xy_in0 += 4 * cst.input_size * cst.batch;
{
#ifdef W_QUANT_4
uchar2 w_int40 = xy_wt[4 * z + 0]; // [N/4, K/4, N4, K4]
FLOAT4 w40 = FLOAT4((float)(w_int40[0] >> 4) - 8, (float)(w_int40[0] & 15) - 8, (float)(w_int40[1] >> 4) - 8, (float)(w_int40[1] & 15) - 8);
#elif defined(W_QUANT_8)
char4 w_int40 = xy_wt[4 * z + 0]; // [N/4, K/4, N4, K4]
FLOAT4 w40 = FLOAT4((float)w_int40[0], (float)w_int40[1], (float)w_int40[2], (float)w_int40[3]);
#endif
FLOAT4 res = w40 * scale[(kl * 2) % 4] + dequant_bias[(kl * 2) % 4];
// [K8, N4, N4]
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 0) * 32 + ml * 4 + kl * 2] = res[0];
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 1) * 32 + ml * 4 + kl * 2] = res[1];
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 2) * 32 + ml * 4 + kl * 2] = res[2];
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 3) * 32 + ml * 4 + kl * 2] = res[3];
}
{
#ifdef W_QUANT_4
uchar2 w_int40 = xy_wt[4 * z + 1]; // [N/4, K/4, N4, K4]
FLOAT4 w40 = FLOAT4((float)(w_int40[0] >> 4) - 8, (float)(w_int40[0] & 15) - 8, (float)(w_int40[1] >> 4) - 8, (float)(w_int40[1] & 15) - 8);
#elif defined(W_QUANT_8)
char4 w_int40 = xy_wt[4 * z + 1]; // [N/4, K/4, N4, K4]
FLOAT4 w40 = FLOAT4((float)w_int40[0], (float)w_int40[1], (float)w_int40[2], (float)w_int40[3]);
#endif
FLOAT4 res = w40 * scale[(kl * 2 + 1) % 4] + dequant_bias[(kl * 2 + 1) % 4];
// [K8, N4, N4]
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 0) * 32 + ml * 4 + kl * 2 + 1] = res[0];
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 1) * 32 + ml * 4 + kl * 2 + 1] = res[1];
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 2) * 32 + ml * 4 + kl * 2 + 1] = res[2];
((threadgroup FLOAT*)sdata)[64 + (kr * 4 + 3) * 32 + ml * 4 + kl * 2 + 1] = res[3];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
simdgroup_load(sga[0], (const threadgroup FLOAT*)sdata, 8);
simdgroup_load(sgb[0], ((const threadgroup FLOAT*)sdata) + 64, 32);
simdgroup_load(sgb[1], ((const threadgroup FLOAT*)sdata) + 72, 32);
simdgroup_load(sgb[2], ((const threadgroup FLOAT*)sdata) + 80, 32);
simdgroup_load(sgb[3], ((const threadgroup FLOAT*)sdata) + 88, 32);
SIMDGROUP_MATRIX_FMA(1, 4);
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
SIMDGROUP_MATRIX_STORE((threadgroup FLOAT*)sdata, 4);
threadgroup_barrier(mem_flags::mem_threadgroup);
if((rx * 8 + (kr*2+kl) * 2) < cst.input_size * cst.batch) {
if((uz * 8 + ml) < cst.output_slice) {
xy_out[0] = activate(ftype4(((threadgroup FLOAT4*)sdata)[((((ml/2) * 4 + (kr*2+kl)) * 2) + 0) * 2 + ml%2] + FLOAT4(biasTerms[uz * 8 + ml])), cst.activation);
}
}
if((rx * 8 + (kr*2+kl) * 2 + 1) < cst.input_size * cst.batch) {
if((uz * 8 + ml) < cst.output_slice) {
xy_out[1] = activate(ftype4(((threadgroup FLOAT4*)sdata)[((((ml/2) * 4 + (kr*2+kl)) * 2) + 1) * 2 + ml%2] + FLOAT4(biasTerms[uz * 8 + ml])), cst.activation);
}
}
}
kernel void conv1x1_gemm_16x16_wquant_sg(const device ftype4 *in [[buffer(0)]],
device ftype4 *out [[buffer(1)]],
constant conv1x1_constants& cst [[buffer(2)]],
@@ -1100,7 +1476,7 @@ kernel void conv1x1_gemv_g4mx_wquant_sg(const device ftype4 *in [[buf
#endif
auto base_xy = xy_in0 + z * area_size;
for(int i = 0; i < AREA_THREAD; i++) {
#ifdef MNN_METAL_SRC_PROTECT
FLOAT4 in40 = (rx + (int)i) < area_size ? (FLOAT4)*(base_xy + i) : (FLOAT4)0;
source/backend/metal/MetalAttention.mm
+4 -4
View File
@@ -114,7 +114,6 @@ void AttentionBufExecution::reallocKVCache() {
return;
}
auto kv_seq_len = mMeta->previous + mMeta->add - mMeta->remove + mMeta->computeReverseSize();
auto mtbn = static_cast<MetalBackend *>(backend());
int byte = 4;
if(mtbn->useFp16InsteadFp32()) {
@@ -125,11 +124,12 @@ void AttentionBufExecution::reallocKVCache() {
// latest length larger than maxLen
if (kv_seq_len > mCache->mMaxLength) {
auto copy_len = mCache->mPastLength - mMeta->remove + mMeta->computeReverseSize();
// copy mPastLength including all remove/reverse to new buffer first
auto copy_len = mCache->mPastLength;
bool needCopy = copy_len > 0;
size_t old_size = mKvNumHead * start * mHeadDim * byte;
size_t old_piece_size = start * byte;
size_t old_size = mKvNumHead * copy_len * mHeadDim * byte;
size_t old_piece_size = copy_len * byte;
size_t old_piece_stride = mCache->mMaxLength * byte;
mCache->mMaxLength = kv_seq_len + mExpandChunk;
source/backend/metal/MetalAttentionShader.hpp
+1 -1
View File
@@ -679,7 +679,7 @@ kernel void softmax_plane_sg(const device ftype *in [[buffer(0)]],
auto axis_out = out + axis_off;
// get max
float max1 = -INFINITY;
float max1 = -FLT_MAX;
for (int i = tiisg; i < s.axis_length; i+=SIMD_GROUP_WIDTH) {
max1 = max(max1, float(axis_in[i * s.inside_size]));
}
source/backend/metal/MetalBackend.hpp
+2 -1
View File
@@ -83,6 +83,8 @@ public:
return &mDynamic[index];
}
BufferAllocator* createDynamicAllocator(int index, bool secondResize) const;
mutable id<MTLCommandBuffer> _waiting = nil;
private:
MetalRuntime(void* context);
void* mContext = nullptr;
@@ -247,7 +249,6 @@ private:
id<MTLComputeCommandEncoder> encoder_net() const;
mutable id<MTLCommandBuffer> _commandBuffer = nil;
mutable id<MTLCommandBuffer> _commandBuffer_net = nil;
mutable id<MTLCommandBuffer> _waiting = nil;
mutable std::queue<id<MTLBuffer>> mHoldBuffers;
id<MTLCommandQueue> _commandQueue;
source/backend/metal/MetalBackend.mm
+6 -7
View File
@@ -94,7 +94,6 @@ MetalBackend::MetalBackend(std::shared_ptr<EagerBufferAllocator> staticMem, cons
}
_commandBuffer = nil;
_commandBuffer_net = nil;
_waiting = nil;
}
MetalBackend::~MetalBackend() {
flushEncoder();
@@ -822,7 +821,7 @@ std::pair<id<MTLBuffer>, int> MetalBackend::getBuffer(const MNN::Tensor* tensor)
void MetalBackend::commit() const {
if (nil != _commandBuffer && _commandBuffer.status < MTLCommandBufferStatusCommitted) {
[_commandBuffer commit];
_waiting = _commandBuffer;
mRuntime->_waiting = _commandBuffer;
_commandBuffer = nil;
if (!mSupportDeferEncode) {
// In this case _commandBuffer should be the same as _commandBuffer_net
@@ -834,7 +833,7 @@ void MetalBackend::commit() const {
void MetalBackend::commit_net() const {
if (nil != _commandBuffer_net && _commandBuffer_net.status < MTLCommandBufferStatusCommitted) {
[_commandBuffer_net commit];
_waiting = _commandBuffer_net;
mRuntime->_waiting = _commandBuffer_net;
_commandBuffer_net = nil;
if (!mSupportDeferEncode) {
// In this case _commandBuffer should be the same as _commandBuffer_net
@@ -844,10 +843,10 @@ void MetalBackend::commit_net() const {
}
void MetalBackend::wait() const {
if (nil != _waiting) {
auto buffer = _waiting;
if (nil != mRuntime->_waiting) {
auto buffer = mRuntime->_waiting;
if (buffer.status >= MTLCommandBufferStatusCompleted) {
_waiting = nil;
mRuntime->_waiting = nil;
return;
}
@@ -872,7 +871,7 @@ void MetalBackend::wait() const {
}
#endif
}
_waiting = nil;
mRuntime->_waiting = nil;
}
id<MTLComputePipelineState> MetalBackend::makeComputePipelineWithSourceOption(const char* csource, const char* cname, MTLCompileOptions *options) const{
source/backend/metal/MetalConvolution1x1.mm
+32 -1
View File
@@ -68,6 +68,7 @@ ErrorCode MetalConvolution1x1::onResize(const std::vector<Tensor *> &inputs, con
for (int i=2; i<input->dimensions(); ++i) {
is *= input->length(i);
}
int ic = input->channel();
int ic_4 = UP_DIV(input->channel(), 4);
int ow = is;
int oh = 1;
@@ -137,7 +138,8 @@ ErrorCode MetalConvolution1x1::onResize(const std::vector<Tensor *> &inputs, con
if (mDequantBits == 4 || mDequantBits == 8) {
// TODO: define short_seq more accurately
int short_seq = 10;
int short_seq = 6;
if(mDequantBits == 4) {
baseKeys.emplace_back("conv1x1_wquant_4");
} else if(mDequantBits == 8) {
@@ -161,6 +163,7 @@ ErrorCode MetalConvolution1x1::onResize(const std::vector<Tensor *> &inputs, con
area = UP_DIV(area, 2);
piece = 2;
}
// MNN_PRINT("Conv1x1 Oc:%d Ic:%d\n", oc, ic_4*4);
std::string kernel_name = "conv1x1_gemv_g4m" + std::to_string(area) + "_wquant_sg";
keys.emplace_back(kernel_name);
auto pipeline = rt->findPipeline(keys);
@@ -231,6 +234,34 @@ ErrorCode MetalConvolution1x1::onResize(const std::vector<Tensor *> &inputs, con
}
mPipeline = pipeline;
mThreads = std::make_pair(MTLSizeMake(UP_DIV(area, 16), UP_DIV(oc, 32), 1), MTLSizeMake(32, 1, 1));
} else if(area < 16) {
// TODO: define useMatrix more accurate
bool useMatrix = area > 6 && oc > 2048 && ic*2 < oc;
if(useMatrix) {
auto keys = baseKeys;
int oc_block = (oc > 4096) ? 32 : 16;
std::string kernel_name = "conv1x1_gemm_8x" + std::to_string(oc_block) + "_wquant_sg";
keys.emplace_back(kernel_name);
auto pipeline = rt->findPipeline(keys);
if (nil == pipeline) {
pipeline = backend->makeComputePipelineWithSourceOption(gConv1x1W4SgMatrix, kernel_name.c_str(), option);
rt->insertPipeline(keys, pipeline);
}
mPipeline = pipeline;
mThreads = std::make_pair(MTLSizeMake(UP_DIV(area, 8), UP_DIV(oc, oc_block), 1), MTLSizeMake(32, 1, 1));
} else {
auto keys = baseKeys;
std::string kernel_name = "conv1x1_gemv_g4m" + std::to_string(area) + "_wquant_sg";
keys.emplace_back(kernel_name);
auto pipeline = rt->findPipeline(keys);
if (nil == pipeline) {
pipeline = backend->makeComputePipelineWithSourceOption(gConv1x1W4SgReduce, kernel_name.c_str(), option);
rt->insertPipeline(keys, pipeline);
}
mPipeline = pipeline;
mThreads = std::make_pair(MTLSizeMake(UP_DIV(oc, 4), 1, 1), MTLSizeMake(32, 1, 1));
}
} else {
auto keys = baseKeys;
keys.emplace_back("conv1x1_gemm_16x16_wquant_sg");
source/backend/metal/MetalLayerNorm.mm
+1 -1
View File
@@ -44,7 +44,7 @@ std::shared_ptr<MetalLayerNorm::Resource> MetalLayerNorm::makeResource(Backend *
if (layernorm->gamma() && layernorm->beta()) {
gamma_size = layernorm->gamma()->size();
}
if (layernorm->external() != nullptr) {
if (layernorm->external() != nullptr && layernorm->external()->size() >= 2) {
auto externalInfo = layernorm->external()->data();
auto externalSize = layernorm->external()->size();
gamma_size = static_cast<int32_t>(externalInfo[1]) / sizeof(float);
source/backend/metal/MetalReduction.hpp
+3 -1
View File
@@ -17,7 +17,7 @@ namespace MNN {
class MetalReduction : public MetalExecution {
public:
MetalReduction(Backend *backend, const ReductionParam *reduction, halide_type_t type);
MetalReduction(Backend *backend, const ReductionParam *reduction);
virtual ~MetalReduction() = default;
virtual ErrorCode onResize(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs) override;
virtual void onEncode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs, id<MTLComputeCommandEncoder> encoder) override;
@@ -27,6 +27,8 @@ private:
id<MTLComputePipelineState> mPipeline;
std::pair<MTLSize, MTLSize> mThreads;
id<MTLBuffer> mConst;
bool mUseSimdReduce = false;
ReductionType mReduceType;
};
} // namespace MNN
source/backend/metal/MetalReduction.mm
+187 -31
View File
@@ -8,46 +8,130 @@
#import "backend/metal/MetalReduction.hpp"
#import "backend/metal/MNNMetalContext.h"
#import "core/Macro.h"
#import "MetalCast.hpp"
#import "core/Macro.h"
#import "backend/metal/MetalBackend.hpp"
#import "core/TensorUtils.hpp"
#if MNN_METAL_ENABLED
namespace MNN {
static const char* gReduceTemplate = R"metal(
#include <metal_stdlib>
#include <simd/simd.h>
using namespace metal;
struct constBuffer
{
// outside_size, axis_size, inside_size, outside_step
int4 size;
};
MetalReduction::MetalReduction(Backend *backend, const ReductionParam *p, halide_type_t type) : MetalExecution(backend) {
auto integer = type.code == halide_type_int;
NSString *kernel;
switch (p->operation()) {
case ReductionType_SUM:
kernel = integer ? @"reduce_sum_s" : @"reduce_sum_f";
break;
case ReductionType_ASUM:
case ReductionType_SUMSQ:
MNN_ASSERT(false); // both un-supported
break;
case ReductionType_MEAN:
kernel = integer ? @"reduce_mean_s" : @"reduce_mean_f";
break;
case ReductionType_MAXIMUM:
kernel = integer ? @"reduce_max_s" : @"reduce_max_f";
break;
case ReductionType_MINIMUM:
kernel = integer ? @"reduce_min_s" : @"reduce_min_f";
break;
case ReductionType_PROD:
kernel = integer ? @"reduce_prod_s" : @"reduce_prod_f";
break;
default:
break;
#define SIMD_GROUP_WIDTH 32
kernel void reduce_shader(const device T* uInput [[buffer(0)]],
device T* uOutput [[buffer(1)]],
constant constBuffer& uConst [[buffer(2)]],
#ifdef SIMD_GROUP_REDUCE
uint3 gid[[threadgroup_position_in_grid]],
uint tiisg[[thread_index_in_simdgroup]],
uint sgitg[[simdgroup_index_in_threadgroup]]
#else
uint3 gid[[thread_position_in_grid]]
#endif
) {
int outside_size = uConst.size.x;
if(gid.x >= outside_size) {
return;
}
int axis_size = uConst.size.y;
int inside_size = uConst.size.z;
int outside_step = uConst.size.w;
auto axis_in = uInput + gid.x * outside_step + gid.y;
#ifdef SIMD_GROUP_REDUCE
#ifdef COMPUTE_REDUCE_MAX
T res = (T)(-FLT_MAX);
for(int i = tiisg; i < axis_size; i+=SIMD_GROUP_WIDTH){
T data = axis_in[i * inside_size];
res = max(res, data);
}
res = simd_max(res);
#elif defined(COMPUTE_REDUCE_SUM)
T res = (T)0;
for(int i = tiisg; i < axis_size; i+=SIMD_GROUP_WIDTH){
T data = axis_in[i * inside_size];
res += data;
}
res = simd_sum(res);
#elif defined(COMPUTE_REDUCE_MEAN)
T res = (T)0;
for(int i = tiisg; i < axis_size; i+=SIMD_GROUP_WIDTH){
T data = axis_in[i * inside_size];
res += data;
}
res = simd_sum(res);
res = res / axis_size;
#elif defined(COMPUTE_REDUCE_MIN)
T res = (T)(FLT_MAX);
for(int i = tiisg; i < axis_size; i+=SIMD_GROUP_WIDTH){
T data = axis_in[i * inside_size];
res = min(res, data);
}
res = simd_min(res);
#elif defined(COMPUTE_REDUCE_PROD)
T res = (T)1;
for(int i = tiisg; i < axis_size; i+=SIMD_GROUP_WIDTH){
T data = axis_in[i * inside_size];
res *= data;
}
res = simd_product(res);
#endif
if(tiisg == 0) {
uOutput[int(gid.x) * inside_size + int(gid.y)] = (T)res;
}
#else
#ifdef COMPUTE_REDUCE_MAX
T res = (T)(-FLT_MAX);
for (int i = 0; i < axis_size; i++) {
T data = axis_in[i * inside_size];
res = max(res, data);
}
#elif defined(COMPUTE_REDUCE_SUM)
M res = (M)0;
for(int i = 0; i < axis_size; i++){
T data = axis_in[i * inside_size];
res += (M)data;
}
#elif defined(COMPUTE_REDUCE_MEAN)
T res = (T)0;
for(int i = 0; i < axis_size; i++){
T data = axis_in[i * inside_size];
res += (M)data;
}
res = res / axis_size;
#elif defined(COMPUTE_REDUCE_MIN)
T res = (T)(FLT_MAX);
for(int i = 0; i < axis_size; i++){
T data = axis_in[i * inside_size];
res = min(res, data);
}
#elif defined(COMPUTE_REDUCE_PROD)
M res = (M)1;
for(int i = 0; i < axis_size; i++){
T data = axis_in[i * inside_size];
res *= (M)data;
}
res = simd_product(res);
#endif
uOutput[int(gid.x) * inside_size + int(gid.y)] = (T)res;
#endif
}
)metal";
MetalReduction::MetalReduction(Backend *backend, const ReductionParam *p) : MetalExecution(backend) {
// The reduce after geometry compute has only one axis
mAxis = p->dim()->data()[0];
mReduceType = p->operation();
auto mkbn = static_cast<MetalBackend *>(backend);
auto context = (__bridge MNNMetalContext *)mkbn->context();
mConst = [context newDeviceBuffer:4 * sizeof(int) access:CPUWriteOnly];
mPipeline = [context pipelineWithName:kernel fp16:mkbn->useFp16InsteadFp32()];
}
ErrorCode MetalReduction::onResize(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs) {
@@ -59,13 +143,85 @@ ErrorCode MetalReduction::onResize(const std::vector<Tensor *> &inputs, const st
for (int i = mAxis + 1; i < inputs[0]->dimensions(); i++) {
insideSize *= inputs[0]->length(i);
}
auto backend = static_cast<MetalBackend *>(this->backend());
auto context = (__bridge MNNMetalContext *)backend->context();
auto mtbn = static_cast<MetalBackend *>(this->backend());
auto context = (__bridge MNNMetalContext *)mtbn->context();
((int *)mConst.contents)[0] = outsideSize;
((int *)mConst.contents)[1] = axisSize;
((int *)mConst.contents)[2] = insideSize;
((int *)mConst.contents)[3] = axisSize * insideSize;
mThreads = [context computeBestGroupAndLocal:mPipeline threads:MTLSizeMake(outsideSize, insideSize, 1)];
bool useFp16 = mtbn->useFp16InsteadFp32();
auto type = inputs[0]->getType();
NSString* T = MetalCast::getScalarType(type, useFp16);
NSString* M = @"float";
if(type.code != halide_type_float) {
M = @"int";
}
std::vector<std::string> keys = {
std::string([T UTF8String]),
std::string([M UTF8String]),
"reduce_shader",
};
switch (mReduceType) {
case ReductionType_SUM:
keys.emplace_back("COMPUTE_REDUCE_SUM");
break;
case ReductionType_ASUM:
case ReductionType_SUMSQ:
MNN_ASSERT(false); // both un-supported
break;
case ReductionType_MEAN:
keys.emplace_back("COMPUTE_REDUCE_MEAN");
break;
case ReductionType_MAXIMUM:
keys.emplace_back("COMPUTE_REDUCE_MAX");
break;
case ReductionType_MINIMUM:
keys.emplace_back("COMPUTE_REDUCE_MIN");
break;
case ReductionType_PROD:
keys.emplace_back("COMPUTE_REDUCE_PROD");
break;
default:
break;
}
if(((MetalRuntime*)mtbn->runtime())->supportSimdGroupReduce()) {
// reduce dimension is large than thread number
if(axisSize > outsideSize * insideSize) {
mUseSimdReduce = true;
}
}
if(mUseSimdReduce) {
keys.emplace_back("SIMD_GROUP_REDUCE");
}
auto pipeline = mtbn->runtime()->findPipeline(keys);
if (nil == pipeline) {
MTLCompileOptions *compileOptions = [[MTLCompileOptions alloc] init];
auto dic = [NSMutableDictionary dictionaryWithCapacity:0];
[dic setValue:T forKey:@"T"];
[dic setValue:M forKey:@"M"];
[dic setValue:@"1" forKey:@(keys[3].c_str())];
if(mUseSimdReduce) {
[dic setValue:@"1" forKey:@"SIMD_GROUP_REDUCE"];
}
compileOptions.preprocessorMacros = dic;
pipeline = mtbn->makeComputePipelineWithSourceOption(gReduceTemplate, "reduce_shader", compileOptions);
mtbn->runtime()->insertPipeline(keys, pipeline);
}
if (nil == pipeline) {
MNN_ERROR("Create gather reduce pipeline error\n");
}
mPipeline = pipeline;
if(mUseSimdReduce) {
mThreads = std::make_pair(MTLSizeMake(outsideSize, insideSize, 1), MTLSizeMake(32, 1, 1));
} else {
mThreads = [context computeBestGroupAndLocal:mPipeline threads:MTLSizeMake(outsideSize, insideSize, 1)];
}
return NO_ERROR;
}
@@ -92,7 +248,7 @@ public:
break;
};
return new MetalReduction(backend, op->main_as_ReductionParam(), inputs[0]->getType());
return new MetalReduction(backend, op->main_as_ReductionParam());
}
};
REGISTER_METAL_OP_CREATOR(MetalReductionCreator, OpType_Reduction);
source/backend/metal/ShaderMap.cpp
-1
View File
@@ -6,7 +6,6 @@ mMaps.insert(std::make_pair("shader_MetalReLU6_metal", shader_MetalReLU6_metal))
mMaps.insert(std::make_pair("shader_MetalConvolutionDepthwise_metal", shader_MetalConvolutionDepthwise_metal));
mMaps.insert(std::make_pair("shader_MetalConvolutionActivation_metal", shader_MetalConvolutionActivation_metal));
mMaps.insert(std::make_pair("shader_MetalConvolution_metal", shader_MetalConvolution_metal));
mMaps.insert(std::make_pair("shader_MetalReduction_metal", shader_MetalReduction_metal));
mMaps.insert(std::make_pair("shader_MetalSoftmax_metal", shader_MetalSoftmax_metal));
mMaps.insert(std::make_pair("shader_MetalLayerNorm_metal", shader_MetalLayerNorm_metal));
mMaps.insert(std::make_pair("shader_MetalConvolutionWinograd_metal", shader_MetalConvolutionWinograd_metal));
source/backend/metal/shader/MetalReduction.metal
-76
View File
@@ -1,76 +0,0 @@
struct reduce_shape {
int outside_size;
int axis_size;
int inside_size;
int outside_step;
};
template <typename M, typename T>
static inline void reduce_mean(const device T *in, device T *out, constant reduce_shape &s, uint2 gid) {
auto axis_in = in + gid.x * s.outside_step + gid.y;
M summer = 0;
for (int i = 0; i < s.axis_size; i++, axis_in += s.inside_size) {
summer += M(*axis_in);
}
out[int(gid.x) * s.inside_size + int(gid.y)] = T(summer / s.axis_size);
}
template <typename M, typename T>
static inline void reduce_sum(const device T *in, device T *out, constant reduce_shape &s, uint2 gid) {
auto axis_in = in + gid.x * s.outside_step + gid.y;
M summer = 0;
for (int i = 0; i < s.axis_size; i++, axis_in += s.inside_size) {
summer += M(*axis_in);
}
out[int(gid.x) * s.inside_size + int(gid.y)] = T(summer);
}
template <typename M, typename T>
static inline void reduce_min(const device T *in, device T *out, constant reduce_shape &s, uint2 gid) {
auto axis_in = in + gid.x * s.outside_step + gid.y;
T summer = *axis_in; axis_in += s.inside_size;
for (int i = 1; i < s.axis_size; i++, axis_in += s.inside_size) {
summer = min(summer, *axis_in);
}
out[int(gid.x) * s.inside_size + int(gid.y)] = summer;
}
template <typename M, typename T>
static inline void reduce_max(const device T *in, device T *out, constant reduce_shape &s, uint2 gid) {
auto axis_in = in + gid.x * s.outside_step + gid.y;
T summer = *axis_in; axis_in += s.inside_size;
for (int i = 1; i < s.axis_size; i++, axis_in += s.inside_size) {
summer = max(summer, *axis_in);
}
out[int(gid.x) * s.inside_size + int(gid.y)] = summer;
}
template <typename M, typename T>
static inline void reduce_prod(const device T *in, device T *out, constant reduce_shape &s, uint2 gid) {
auto axis_in = in + gid.x * s.outside_step + gid.y;
M summer = 1;
for (int i = 0; i < s.axis_size; i++, axis_in += s.inside_size) {
summer *= M(*axis_in);
}
out[int(gid.x) * s.inside_size + int(gid.y)] = T(summer);
}
#define define_reduce(name) \
kernel void reduce_##name##_f(const device ftype *in [[buffer(0)]], \
device ftype *out [[buffer(1)]], \
constant reduce_shape &s [[buffer(2)]], \
uint2 gid [[thread_position_in_grid]]) { \
if (gid.x < (uint)s.outside_size && gid.y < (uint)s.inside_size) reduce_##name<FLOAT, ftype>(in, out, s, gid); \
} \
kernel void reduce_##name##_s(const device int *in [[buffer(0)]], \
device int *out [[buffer(1)]], \
constant reduce_shape &s [[buffer(2)]], \
uint2 gid [[thread_position_in_grid]]) { \
if (gid.x < (uint)s.outside_size && gid.y < (uint)s.inside_size) reduce_##name<int, int>(in, out, s, gid); \
}
define_reduce(mean);
define_reduce(sum);
define_reduce(min);
define_reduce(max);
define_reduce(prod);
source/backend/metal/shader/MetalSoftmax.metal
+1 -1
View File
@@ -28,7 +28,7 @@ kernel void softmax_plane(const device ftype *in [[buffer(0)]],
auto axis_out = out + axis_off;
// get max
float max1 = -INFINITY;
float max1 = -FLT_MAX;
for (int i = 0; i < s.axis_length; i++) {
max1 = max(max1, float(axis_in[i * s.inside_size]));
}
source/backend/neuropilot/CMakeLists.txt
+17
View File
@@ -0,0 +1,17 @@
set(MNN_NEUROPILOT_CONVERT_MODE ON)
if(CMAKE_SYSTEM_NAME MATCHES "^Android")
set(MNN_NEUROPILOT_CONVERT_MODE OFF)
endif()
file(GLOB BACKEND_SRCS ${CMAKE_CURRENT_LIST_DIR}/backend/*)
file(GLOB CONVERTER_SRCS ${CMAKE_CURRENT_LIST_DIR}/converter/*)
file(GLOB_RECURSE PLUGIN_SRCS ${CMAKE_CURRENT_LIST_DIR}/mtk/*)
if (MNN_NEUROPILOT_CONVERT_MODE)
add_library(MNN_NEUROPILOT OBJECT ${BACKEND_SRCS} ${CONVERTER_SRCS})
target_compile_options(MNN_NEUROPILOT PRIVATE -DMNN_NEUROPILOT_CONVERT_MODE)
else()
include_directories(${CMAKE_CURRENT_LIST_DIR}/mtk)
add_library(MNN_NEUROPILOT OBJECT ${BACKEND_SRCS} ${PLUGIN_SRCS})
endif()
target_include_directories(MNN_NEUROPILOT PRIVATE ${CMAKE_CURRENT_LIST_DIR}/)
source/backend/neuropilot/README.md
+175
View File
@@ -0,0 +1,175 @@
# 导出并量化模型
## 编译 MNN
```plaintext
mkdir build
cmake ../ -DMNN_BUILD_CONVERTER=ON -DMNN_BUILD_LLM=ON -DMNN_NEUROPILOT=ON
make -j4
```
准备测试文件 512.txt
```plaintext
我在写一个以魔法世界为主题的中篇小说。我已经完成了开头，请你在我完成的部分的基础上续写该小说。我已经完成的部分如下：<<<《奥术回廊的第七根蜡烛》第一章：褪色的录取函1. 灰烬中的秘密伊莱·维瑟兰用拇指摩挲着信封边缘的金色火漆印，那枚印记在他触碰的瞬间微微发烫，仿佛活物般收缩了一下。他犹豫片刻，还是撕开了信封。羊皮纸在展开的刹那化作细碎的灰烬，像一群受惊的飞蛾般四散飘落，只留下几行暗红色的字迹浮现在空中：“以三滴晨露与午夜叹息为凭，汝已被奥术回廊学院录取。”“报到日：下弦月之夜。”“携带物品：一根未染血的银针、一颗自愿献出的牙齿、一段不为人知的记忆。”伊莱皱眉，指尖轻轻触碰最后一行几乎被烧毁的小字——那里本该是校训的位置，却只剩下半句残缺的警告：“知识是……”后面的字迹被某种力量抹去，只留下一道焦黑的裂痕，像是被火焰舔舐过的皮肤。“知识是蜜糖还是毒药？”母亲的声音突然从身后传来，枯瘦的手指轻轻搭上他的肩膀。伊莱猛地回头，发现她不知何时站在了阴影里，烛光只照亮她半边脸，另一半隐没在黑暗中，像是被什么东西啃噬过。“你父亲收到录取函时，最后一行是完整的。”她低声说，指尖滑向伊莱后颈的胎记——那团火焰形状的印记此刻正隐隐发烫，就像七岁那年他无意间触碰祖父的魔法书时一样。“别让他们发现你能看见‘不该看的东西’。”她最后叮嘱道，声音轻得像一阵风，随后转身消失在走廊尽头，只留下一缕若有若无的草药苦味。2. 雨中学院报到日当天，暴雨倾盆。奥术回廊学院矗立在悬崖边缘，哥特式的尖顶刺破铅灰色的天空，黑曜石外墙不断渗出粘稠的黑色液体，像是一头受伤的巨兽在流血。新生们踩着骨白色的台阶向上攀登。...>>>
```
## 导出
有两种方案
### 使用 smoothquant
增加 `--smooth --act_bit=16 --quant_block=0 --lm_quant_bit=16 --quant_bit=4 --seperate_embed --sym` 以导出 mnn
eg: 
```
python3 llmexport.py --path /Users/xtjiang/.cache/modelscope/hub/models/Qwen/Qwen3-4B --export mnn --smooth --act_bit=16 --quant_block=0 --lm_quant_bit=16 --seperate_embed --quant_bit=4 --sym
```
### 两步量化（相比前者快很多，但目前效果较差）
- 使用 `--quant_block=0 --lm_quant_bit=16 --seperate_embed --quant_bit=4 --sym` 导出 mnn 
- 执行 `./quantize_llm ../transformers/llm/export/model/config.json 512.txt 16 temp.bin && cp temp.bin ../transformers/llm/export/model/llm.mnn` 量化特征
# 使用 `compilefornpu` 生成 tflite
## 使用 generateLlmIO 生成 input / output
```
./generateLlmIO ../transformers/llm/export/model  ../transformers/llm/export/model/testdir
```
## 编辑如下的json文件：npu.json
```json
{
"name":"MLDA",
"skips":[
"/Reshape_output_0",
"/Gather_3_output_0",
"/Gather_4_output_0"
],
"testdir":[
"testdir/1",
"testdir/128"
]
}
```
将 testdir 中的路径修改为 model/testdir 对应路径
##  执行 `compilefornpu`
rm -r res
mkdir res
./seperatenpu ../transformers/llm/export/model/llm.mnn res/temp.bin npu.json
当前目录下会增加 `npu_postreat.json`
# 使用 `npu_convert.py` 将 tflite 编译为 dla
## 下载 sdk 并配置环境变量
下载 NEURON_SDK ，并修改 `~/.bashrc` ，增加对应的路径
eg:
```
export NEURON_SDK=/home/xiaying/third/mtk/neuropilot-sdk-basic-7.0.8-build20240807/neuron_sdk
```
## 执行转换脚本
```
python3 ../source/backend/neuropilot/npu_convert.py npu_postreat.json > 1
```
## 【可选】清除中间产物
```
rm res/*.tflite
rm res/*.dla
```
# model 目录构建
在 build 目录下执行
```
rm -r model
mv ../transformer/llm/export/model model
```
## npu 相关文件复制
将 res/ 下的构建产物复制到 model 目录下，并用 res/temp.bin 重命名为 model/llm_npu.mnn 
```plaintext
rm -r model/res
mv res model/res
mv model/res/temp.bin model/llm_npu.mnn
```
## 增加 `model/config_npu.json`
```json
{
"llm_model": "llm_npu.mnn",
"backend_type": "cpu",
"thread_num": 4,
"precision": "low",
"chunk_limits":[128, 1],
"memory": "low",
"sampler_type": "penalty",
"penalty": 1.1
}
```
## 将 model/llm.mnn.weight 删除（可选）
* llm.mnn 和 llm.mnn.weight 不再需要，如果不需要对比 CPU / GPU 的性能可以将它们删除
```plaintext
rm model/llm.mnn
rm model/llm.mnn.weight
```
# 测试
## 测试资源准备
* 将构建产物传到设备上：
```plaintext
#!/bin/bash
adb shell mkdir /data/local/tmp/MNN
adb shell rm -r /data/local/tmp/MNN/model
adb push model /data/local/tmp/MNN/model
```
* 上传测试文件 512.txt
```
adb push 512.txt /data/local/tmp/MNN/512.txt
```
## 运行测试程序
* 编译并上传
在 MNN 根目录下逐句操作：
```plaintext
cd project/android/
mkdir build_64
cd build_64
../build_64.sh -DMNN_NEUROPILOT=ON -DMNN_WITH_PLUGIN=ON -DMNN_BUILD_LLM=ON
../updateTest.sh
```
* 运行程序
```plaintext
cd project/android/build_64
../testCommon.sh ./llm_demo model/config_npu.json 512.txt
```
source/backend/neuropilot/backend/NeuropilotBackend.cpp
+851
View File
@@ -0,0 +1,851 @@
#include "NeuropilotBackend.hpp"
#ifdef MNN_NEUROPILOT_CONVERT_MODE
#include "converter/ConvertExecution.hpp"
#include "converter/OptimizeCommandBuffer.hpp"
#include "core/Macro.h"
#include "core/TensorUtils.hpp"
#include "MNN/MNNDefine.h"
#include <fstream>
#include <iostream>
#endif
#ifndef MNN_NEUROPILOT_CONVERT_MODE
#ifdef MNN_WITH_PLUGIN
#include "flatbuffers/flexbuffers.h"
#include "MNN/plugin/PluginShapeInference.hpp"
#include "MNN/plugin/PluginContext.hpp"
#include "MNN/plugin/PluginKernel.hpp"
#include "shape/SizeComputer.hpp"
#include "common/file_source.h"
#include "executor/neuron_usdk_executor.h"
#include "backend/arm82/Arm82OptFunc.hpp"
#include "backend/cpu/CPUBackend.hpp"
#include "core/OpCommonUtils.hpp"
#endif
#endif
namespace MNN {
#ifdef MNN_NEUROPILOT_CONVERT_MODE
struct TensorInfo {
std::unique_ptr<tflite::TensorT> tensor;
std::unique_ptr<tflite::BufferT> buffer;
bool isInput = false;
bool isOutput = false;
};
void NeuropilotBackend::prepareTensorQuantInfo(const Tensor* tensor, std::unique_ptr<tflite::QuantizationParametersT>&& param) {
mQuantInfo.insert(std::make_pair(tensor, std::move(param)));
}
void NeuropilotBackend::setPackTensor(const Tensor* tensor, int packBits) {
mPackInfo.insert(std::make_pair(tensor, packBits));
}
void NeuropilotBackend::setTensorName(const Tensor* tensor, std::string name) {
mUserTensorName.insert(std::make_pair(tensor, name));
}
void NeuropilotBackend::insertExtraInput(Tensor* tensor) {
mExtraInputs.insert(std::make_pair(tensor, mExtraInputs.size()));
}
void NeuropilotBackend::insertExtraOutput(Tensor* tensor) {
mExtraOutputs.insert(std::make_pair(tensor, mExtraOutputs.size()));
}
int NeuropilotBackend::_createTensorFromMNNTensor(const Tensor* tensor, tflite::SubGraphT* dstGraph, std::vector<std::unique_ptr<tflite::BufferT>>& dstBuffers) {
if (mTensorIndexMap.find(tensor) != mTensorIndexMap.end()) {
return mTensorIndexMap[tensor];
}
TensorInfo info;
info.tensor.reset(new tflite::TensorT);
info.buffer.reset(new tflite::BufferT);
auto& tfliteTensor = info.tensor;
int tensorIndex = (int)mTensorIndexMap.size();
mTensorIndexMap[tensor] = tensorIndex;
auto des = TensorUtils::getDescribe(tensor);
do {
if (mStateMask.get() == tensor) {
tfliteTensor->name = "mask";
break;
}
if (mExtraInputs.find(tensor) != mExtraInputs.end()) {
tfliteTensor->name = "ei" + std::to_string(mExtraInputs.find(tensor)->second);
break;
}
if (mExtraOutputs.find(tensor) != mExtraOutputs.end()) {
tfliteTensor->name = "eo" + std::to_string(mExtraOutputs.find(tensor)->second);
break;
}
if (mUserTensorName.find(tensor) == mUserTensorName.end()) {
if (des->index >= 0) {
tfliteTensor->name = "t" + std::to_string(des->index);
} else {
tfliteTensor->name = "tensor_" + std::to_string(tensorIndex);
}
} else {
tfliteTensor->name = mUserTensorName[tensor];
}
if (des->usage == Tensor::InsideDescribe::Usage::INPUT) {
info.isInput = true;
}
if (des->usage == Tensor::InsideDescribe::Usage::OUTPUT) {
info.isOutput = true;
if (des->applyQuant) {
mDequantTensor.insert(std::make_pair(tensorIndex, tensor));
}
}
} while (false);
tfliteTensor->type = ConvertTflite::getType(tensor);
if (des->usage == Tensor::InsideDescribe::Usage::CONSTANT) {
if (des->dimensionFormat == MNN_DATA_FORMAT_NC4HW4) {
MNN_ERROR("Don't support NC4HW4 constant now\n");
}
auto size = tensor->usize();
if (mPackInfo.find(tensor) != mPackInfo.end()) {
tfliteTensor->type = tflite::TensorType_INT4;
auto buffer_size = size / 2;
info.buffer->data.resize(buffer_size);
auto src_buffer = tensor->host<int8_t>();
// Ref from kernels/test_util.h
// Funciton: PackInt4ValuesDenselyInPlace
for (int i = 0; i < size; ++i) {
int srcValue = src_buffer[i];
if (i % 2 == 0) {
info.buffer->data[i / 2] = srcValue & 0x0F;
} else {
info.buffer->data[i / 2] |= srcValue << 4;
}
}
} else {
info.buffer->data.resize(size);
::memcpy(info.buffer->data.data(), tensor->host<void>(), size);
}
}
if (mQuantInfo.find(tensor) != mQuantInfo.end()) {
tfliteTensor->quantization = std::move(mQuantInfo.find(tensor)->second);
mQuantInfo.erase(tensor);
} else if (des->applyQuant && des->quantAttr.get() != nullptr) {
// Load quant info
auto scale = des->quantAttr->scale;
auto zero = des->quantAttr->zero;
tfliteTensor->quantization.reset(new tflite::QuantizationParametersT);
tfliteTensor->quantization->scale = {scale};
tfliteTensor->quantization->zero_point = {(long)zero};
tfliteTensor->quantization->max = {(des->quantAttr->max-zero) * scale};
tfliteTensor->quantization->min = {(des->quantAttr->min-zero) * scale};
}
tfliteTensor->shape = ConvertTflite::getShapeOfTensor(tensor);
tfliteTensor->buffer = (int)dstBuffers.size();
dstBuffers.emplace_back(std::move(info.buffer));
int dstTensorIndex = (int)dstGraph->tensors.size();
dstGraph->tensors.emplace_back(std::move(info.tensor));
if (info.isInput) {
dstGraph->inputs.emplace_back(des->index);
if (des->index < 0) {
MNN_ERROR("Invalid Input Tensor for construct Tflite\n");
}
mIOIndexMap.insert(std::make_pair(des->index, dstTensorIndex));
}
if (info.isOutput) {
if (des->index < 0) {
MNN_ERROR("Invalid Output Tensor for construct Tflite\n");
}
dstGraph->outputs.emplace_back(des->index);
mIOIndexMap.insert(std::make_pair(des->index, dstTensorIndex));
}
return tensorIndex;
}
Backend* NeuropilotRuntime::onCreate(const BackendConfig* config, Backend* origin) const {
return new NeuropilotBackend(this);
}
Execution* NeuropilotBackend::onCreate(const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, const MNN::Op* op) {
return new ConvertExecution(this, op);
}
void NeuropilotBackend::onResizeBegin() {
mInfos.clear();
mTensorIndexMap.clear();
mIOIndexMap.clear();
mPackInfo.clear();
mDequantTensor.clear();
mExtraInputs.clear();
mExtraOutputs.clear();
mStateMask = nullptr;
mSharedConst.clear();
}
Tensor* NeuropilotBackend::getConstTensor(std::string name, std::shared_ptr<Tensor> ref) {
if (mSharedConst.find(name) == mSharedConst.end()) {
if (ref == nullptr) {
return nullptr;
}
mSharedConst.insert(std::make_pair(name, ref));
return ref.get();
}
return mSharedConst.find(name)->second.get();
}
Tensor* NeuropilotBackend::getStateMask(int maxLength) {
if (nullptr == mStateMask) {
mStateMask.reset(Tensor::createDevice<float>({maxLength}));
}
return mStateMask.get();
}
ErrorCode NeuropilotBackend::onResizeEnd() {
// 如果没有操作信息，直接返回
if (mInfos.empty()) {
MNN_PRINT("NeuropilotBackend::onResizeEnd: No operations to convert\n");
return NO_ERROR;
}
MNN_PRINT("NeuropilotBackend::onResizeEnd: Converting %zu operations to TensorFlow Lite model\n", mInfos.size());
// 创建 TensorFlow Lite 模型
std::unique_ptr<tflite::ModelT> tfliteModel = createTensorFlowLiteModel();
if (tfliteModel) {
MNN_PRINT("Successfully created TensorFlow Lite model with %zu subgraphs and %zu operator codes\n",
tfliteModel->subgraphs.size(), tfliteModel->operator_codes.size());
// 保存模型到缓存路径（如果有的话）
if (mRuntime && !mRuntime->pCachePath.empty()) {
std::string modelPath = mRuntime->pCachePath;
saveTensorFlowLiteModel(tfliteModel, modelPath);
MNN_PRINT("TensorFlow Lite model conversion completed successfully\n");
} else {
MNN_PRINT("TensorFlow Lite model created but no cache path specified for saving\n");
}
} else {
MNN_ERROR("Failed to create TensorFlow Lite model\n");
return COMPUTE_SIZE_ERROR;
}
return NO_ERROR;
}
Backend::MemObj* NeuropilotBackend::onAcquire(const Tensor* tensor, StorageType storageType) {
return new Backend::MemObj;
}
class NeuropilotRuntimeCreator : public RuntimeCreator {
public:
virtual Runtime* onCreate(const Backend::Info& info) const override {
return new NeuropilotRuntime(info);
}
virtual bool onValid(Backend::Info& info) const override {
return true;
}
static bool _supportQuant(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs) {
auto otype = op->type();
switch (otype) {
case OpType_Convolution:
case OpType_ConvolutionDepthwise:
{
if (inputs.size() > 1) {
return false;
}
auto conv2d = op->main_as_Convolution2D();
if (nullptr != conv2d && nullptr != conv2d->quanParameter() && conv2d->quanParameter()->type() == 1) {
return true;
}
return false;
}
case OpType_Pooling:
case OpType_Reshape:
case OpType_Transpose:
case OpType_ConvertTensor:
case OpType_Flatten:
case OpType_Squeeze:
case OpType_Unsqueeze:
case OpType_Slice:
case OpType_StridedSlice:
case OpType_Identity:
{
auto input = inputs[0];
if (input->getType().code != halide_type_float || TensorUtils::getDescribe(input)->quantAttr.get() != TensorUtils::getDescribe(outputs[0])->quantAttr.get()) {
return false;
}
}
return true;
default:
break;
}
return false;
}
virtual bool onSetQuantInfo(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs) const override {
if (nullptr == op) {
return true;
}
auto res = _supportQuant(op, inputs, outputs);
for (auto t : outputs) {
TensorUtils::getDescribe(t)->applyQuant = res;
}
return res;
}
};
// TensorFlow Lite 转换相关的辅助函数实现
std::unique_ptr<tflite::ModelT> NeuropilotBackend::createTensorFlowLiteModel() {
std::unique_ptr<tflite::ModelT> model(new tflite::ModelT);
model->version = 3; // TensorFlow Lite 版本
model->description = "Converted from MNN model";
std::unique_ptr<tflite::SubGraphT> subgraph(new tflite::SubGraphT);
subgraph->name = "main";
mTensorIndexMap.clear();
int tensorIndex = 0;
std::vector<std::unique_ptr<tflite::OperatorT>> operators;
ConvertTflite converter;
converter.pBackend = this;
{
std::unique_ptr<tflite::BufferT> buf(new tflite::BufferT);
model->buffers.emplace_back(std::move(buf));
}
ConvertTflite::CommandBuffer totalCmdBuffer;
for (const auto& info : mInfos) {
auto cmdbuffer = converter.convert(info.op, info.inputs, info.outputs);
totalCmdBuffer.extraConst.insert(totalCmdBuffer.extraConst.end(), cmdbuffer.extraConst.begin(), cmdbuffer.extraConst.end());
for (auto&& cmd : cmdbuffer.commands) {
totalCmdBuffer.commands.emplace_back(std::move(cmd));
}
}
for (auto& t : totalCmdBuffer.extraConst) {
_createTensorFromMNNTensor(t.get(), subgraph.get(), model->buffers);
}
OptimizeCommandBuffer opt(&converter);
totalCmdBuffer = opt.reduce(std::move(totalCmdBuffer));
for (auto& cmd : totalCmdBuffer.commands) {
if (cmd.op.get() == nullptr) {
continue;
}
for (const auto& inputTensor : cmd.inputs) {
_createTensorFromMNNTensor(inputTensor, subgraph.get(), model->buffers);
}
for (const auto& inputTensor : cmd.outputs) {
_createTensorFromMNNTensor(inputTensor, subgraph.get(), model->buffers);
}
auto op = std::move(cmd.op);
for (const auto& inputTensor : cmd.inputs) {
op->inputs.push_back(mTensorIndexMap[inputTensor]);
}
for (const auto& outputTensor : cmd.outputs) {
op->outputs.push_back(mTensorIndexMap[outputTensor]);
}
operators.push_back(std::move(op));
}
std::vector<std::unique_ptr<tflite::OperatorT>> deqOp;
std::vector<std::shared_ptr<Tensor>> deqTensors;
for (auto& iter : mDequantTensor) {
auto originIndex = iter.first;
std::shared_ptr<Tensor> floatTensor(new Tensor(iter.second, iter.second->getDimensionType(), false));
TensorUtils::getDescribe(floatTensor.get())->dimensionFormat = TensorUtils::getDescribe(iter.second)->dimensionFormat;
deqTensors.emplace_back(floatTensor);
auto newIndex = _createTensorFromMNNTensor(floatTensor.get(), subgraph.get(), model->buffers);
// Swap tensor info
auto swapT = std::move(subgraph->tensors[newIndex]);
subgraph->tensors[newIndex] = std::move(subgraph->tensors[originIndex]);
subgraph->tensors[originIndex] = std::move(swapT);
// Swap name
auto name = subgraph->tensors[newIndex]->name;
subgraph->tensors[newIndex]->name = subgraph->tensors[originIndex]->name;
subgraph->tensors[originIndex]->name = name;
// Reset op output index
for (auto& op : operators) {
for (int i=0; i<op->inputs.size(); ++i) {
if (op->inputs[i] == originIndex) {
op->inputs[i] = newIndex;
}
}
for (int i=0; i<op->outputs.size(); ++i) {
if (op->outputs[i] == originIndex) {
op->outputs[i] = newIndex;
}
}
}
// Add Dequant Op
std::unique_ptr<tflite::OperatorT> dequantOp(new tflite::OperatorT);
dequantOp->opcode_index = converter.getOpIndex(tflite::BuiltinOperator_DEQUANTIZE);
dequantOp->inputs = {newIndex};
dequantOp->outputs = {originIndex};
deqOp.emplace_back(std::move(dequantOp));
}
for (auto&& op : deqOp) {
operators.emplace_back(std::move(op));
}
subgraph->operators = std::move(operators);
// Reindex subgraph io
std::sort(subgraph->inputs.begin(), subgraph->inputs.end());
std::sort(subgraph->outputs.begin(), subgraph->outputs.end());
for (int i=0; i<subgraph->inputs.size(); ++i) {
subgraph->inputs[i] = mIOIndexMap[subgraph->inputs[i]];
}
for (int i=0; i<subgraph->outputs.size(); ++i) {
subgraph->outputs[i] = mIOIndexMap[subgraph->outputs[i]];
}
// Insert Extra Input and Output
if (nullptr != mStateMask) {
auto index = mTensorIndexMap.find(mStateMask.get())->second;
subgraph->inputs.emplace_back(index);
}
for (int i=0; i<mExtraInputs.size(); ++i) {
for (auto& iter : mExtraInputs) {
if (iter.second == i) {
auto index = mTensorIndexMap.find(iter.first)->second;
subgraph->inputs.emplace_back(index);
break;
}
}
}
for (int i=0; i<mExtraOutputs.size(); ++i) {
for (auto& iter : mExtraOutputs) {
if (iter.second == i) {
auto index = mTensorIndexMap.find(iter.first)->second;
subgraph->outputs.emplace_back(index);
break;
}
}
}
model->subgraphs.push_back(std::move(subgraph));
model->operator_codes = converter.releaseCodes();
return model;
}
void NeuropilotBackend::saveTensorFlowLiteModel(std::unique_ptr<tflite::ModelT>& model, const std::string& filePath) {
flatbuffers::FlatBufferBuilder builder;
builder.ForceDefaults(true);
auto modelOffset = tflite::Model::Pack(builder, model.get());
// Must use tflite::FinishModelBuffer, otherwise can't verify success
tflite::FinishModelBuffer(builder, modelOffset);
std::ofstream file(filePath, std::ios::binary);
if (file.is_open()) {
file.write(reinterpret_cast<const char*>(builder.GetBufferPointer()), builder.GetSize());
file.close();
MNN_PRINT("TensorFlow Lite model saved to: %s\n", filePath.c_str());
} else {
MNN_ERROR("Failed to save TensorFlow Lite model to: %s\n", filePath.c_str());
}
}
#endif
#ifndef MNN_NEUROPILOT_CONVERT_MODE
#ifdef MNN_WITH_PLUGIN
namespace plugin {
static bool computeIndex(const std::vector<Tensor *> & inputs, const Attribute* attrAllShape, int & index) {
if (nullptr == attrAllShape || nullptr == attrAllShape->list() || nullptr == attrAllShape->list()->i()) {
MNN_ERROR("[NeuropilotBackend] input shape attr error\n");
return false;
}
int dimSum = 0;
for (int i = 0; i < inputs.size(); i++) {
auto inputDim = inputs[i]->dimensions();
dimSum += inputDim;
}
if (0 == dimSum) {
// Scalar
index = 0;
return true;
}
auto indexNumber = attrAllShape->list()->i()->size() / dimSum;
for (int si=0; si<indexNumber; ++si) {
auto dstSi = attrAllShape->list()->i()->data() + si * dimSum;
bool valid = true;
for (int i=0; i<inputs.size(); ++i) {
auto inputDim = inputs[i]->dimensions();
for (int j = 0; j < inputDim; j++) {
if (inputs[i]->length(j) != dstSi[j]) {
valid = false;
break;
}
}
dstSi += inputDim;
if (!valid) {
break;
}
}
if (valid) {
index = si;
return true;
}
}
return false;
}
static std::vector<std::vector<int>> _extractShapes(const int* shape, const std::vector<Tensor*>& inputs) {
// Divide by "_"
std::vector<std::vector<int>> res;
for (auto s : inputs) {
std::vector<int> dim(s->dimensions());
for (int i=0; i<dim.size(); ++i) {
dim[i] = shape[i];
}
shape += s->dimensions();
res.emplace_back(dim);
}
return res;
}
namespace shape_inference {
class PluginShapeRaw : public InferShapeKernel {
public:
bool compute(InferShapeContext* ctx) override;
};
bool PluginShapeRaw::compute(InferShapeContext* ctx) {
if (ctx->hasAttr("op")) {
auto attr = ctx->getAttr("op");
if (nullptr != attr->tensor() && nullptr != attr->tensor()->int8s()) {
auto realop = flatbuffers::GetRoot<Op>(attr->tensor()->int8s()->data());
return SizeComputer::computeOutputSize(realop, ctx->inputs(), ctx->outputs());
}
} else {
int shapeIndex = 0;
auto attrAllShape = ctx->getAttr("allInputShape");
if (!(computeIndex(ctx->inputs(), attrAllShape, shapeIndex))) {
MNN_ERROR("Failed to compute shape for Plugin Op.\n");
return false;
}
std::string prefix = "o_" + std::to_string(shapeIndex) + "_";
for (int i=0; i<ctx->outputs().size(); ++i) {
auto dst = ctx->output(i);
std::string key = prefix + std::to_string(i);
auto attr = ctx->getAttr(key.c_str());
if (nullptr == attr || nullptr == attr->tensor()) {
MNN_ERROR("MNN_QNN: Failed to find raw shape %s.\n", key.c_str());
return false;
}
auto blob = attr->tensor();
dst->setType(blob->dataType());
if (nullptr != blob->dims()) {
dst->buffer().dimensions = blob->dims()->size();
for (int j=0; j<blob->dims()->size(); ++j) {
dst->setLength(j, blob->dims()->data()[j]);
}
} else {
dst->buffer().dimensions = 0;
}
TensorUtils::getDescribe(dst)->dimensionFormat = blob->dataFormat();
}
return true;
}
return false;
}
}
namespace backend {
class PluginExecuteRaw : public CPUComputeKernel {
private:
std::vector<std::pair<const MNN::Tensor *, size_t>> mInputs;
std::vector<std::pair<const MNN::Tensor *, size_t>> mOutputs;
std::shared_ptr<mtk::NeuronUsdkExecutor> mExecutor;
std::unique_ptr<mtk::SharedWeightsHandle> mSharedWeightsHandle;
std::vector<std::shared_ptr<mtk::NeuronUsdkExecutor>> mAllExecutors;
std::string mPath;
struct StateTensor {
mtk::IOBuffer data;
int inside;
int outside;
};
std::vector<StateTensor> mStateInput;
int mStateCurrent = 0;
int mStateMaxSize = 0;
mtk::IOBuffer mMask;
bool mMaskFp16 = false;
const float mMinValue = -32700.0f;
public:
~ PluginExecuteRaw() {
mExecutor.reset();
}
void _loadState(mtk::NeuronUsdkExecutor* executor, int inputSize, int stateNumber) {
if (stateNumber == 0) {
return;
}
mMask = executor->getInput(inputSize);
if (mMask.sizeBytes == mStateMaxSize * sizeof(__fp16)) {
mMaskFp16 = true;
}
// Init Mask
if (mMaskFp16) {
auto dstM = (__fp16*)mMask.buffer;
for (int i=0; i<mStateMaxSize; ++i) {
dstM[i] = mMinValue;
}
} else {
auto dstM = (float*)mMask.buffer;
for (int i=0; i<mStateMaxSize; ++i) {
dstM[i] = mMinValue;
}
}
for (int i=0; i<mStateInput.size(); ++i) {
mStateInput[i].data = executor->getInput(inputSize + 1 + i);
}
}
bool init(CPUKernelContext* ctx) override {
auto state = ctx->getAttr("state");
mNeedAllocIO = false;
int stateNumber = 0;
if (nullptr != state) {
int axis = 0;
auto ref = flexbuffers::GetRoot(state->tensor()->uint8s()->data(), state->tensor()->uint8s()->size());
auto refMap = ref.AsMap();
auto keys = refMap.Keys();
std::vector<std::vector<int>> stateShape;
for (int i=0; i<keys.size(); ++i) {
auto key = keys[i].AsKey();
if (std::string(key) == "number") {
stateNumber = refMap.Values()[i].AsInt32();
continue;
}
if (std::string(key) == "max_length") {
mStateMaxSize = refMap.Values()[i].AsInt32();
continue;
}
if (std::string(key) == "axis") {
axis = refMap.Values()[i].AsInt32();
continue;
}
if (std::string(key) == "shape") {
auto shapeVectors = refMap.Values()[i].AsVector();
for (int u=0; u<shapeVectors.size(); ++u) {
auto shapeV = shapeVectors[u].AsVector();
std::vector<int> shapes;
for (int v=0; v<shapeV.size(); ++v) {
shapes.emplace_back(shapeV[v].AsInt32());
}
stateShape.emplace_back(shapes);
}
continue;
}
}
mStateInput.resize(stateShape.size());
for (int i=0; i<stateShape.size(); ++i) {
auto& shape = stateShape[i];
auto& input = mStateInput[i];
input.outside = 1;
for (int j=0; j<axis; ++j) {
input.outside *= shape[j];
}
auto axisLength = shape[axis];
MNN_ASSERT(1 == axisLength);
input.inside = 1;
for (int j=axis+1; j<shape.size(); ++j) {
input.inside *= shape[j];
}
}
}
FUNC_PRINT(stateNumber);
int maskNumber = stateNumber > 0 ? 1 : 0;
auto allGraph = ctx->getAttr("allGraphName");
auto inputTensor = ctx->inputs();
auto outputTensor = ctx->outputs();
mPath = ctx->getAttr("path")->s()->str();
if (nullptr != allGraph->list() && nullptr != allGraph->list()->s() && allGraph->list()->s()->size() > 1) {
int allGraphNumber = allGraph->list()->s()->size();
// ShareWeight
{
auto path = ctx->dir_path() + mPath + ".weight";
FileSource files(path.c_str());
mSharedWeightsHandle.reset(new mtk::SharedWeightsHandle({files}, 1));
mSharedWeightsHandle->preload();
}
auto ShareWeight = mSharedWeightsHandle->getSharedWeights(0);
// AllExecutor
mAllExecutors.resize(allGraphNumber);
auto allInputShape = ctx->getAttr("allInputShape");
int inputShapeLength = allInputShape->list()->i()->size() / allGraphNumber;
for (int i = 0; i < allGraphNumber; ++i) {
auto shape = _extractShapes(allInputShape->list()->i()->data() + i * inputShapeLength, inputTensor);
auto path = ctx->dir_path() + mPath + ".shared_" + std::to_string(i);
FileSource files(path.c_str());
// Input + mask + state
int inputSize = inputTensor.size() + maskNumber + stateNumber;
int validInputSize = inputTensor.size();
for (auto& s : shape) {
int size = 1;
for (auto d : s) {
size *= d;
}
if (1 == size) {
inputSize -= 1;
validInputSize -= 1;
}
}
mAllExecutors[i].reset(new mtk::NeuronUsdkExecutor(inputSize, files, ShareWeight));
// Add shared weight input
mAllExecutors[i]->setNumInputs(inputSize+1);
mAllExecutors[i]->setNumOutputs(outputTensor.size()+stateNumber);
if (0 != i && stateNumber > 0) {
mAllExecutors[i]->setModelInput(mMask, validInputSize);
for (int j=0; j<mStateInput.size(); ++j) {
mAllExecutors[i]->setModelInput(mStateInput[j].data, validInputSize+1+j);
}
}
mAllExecutors[i]->initialize();
if (0 == i) {
_loadState(mAllExecutors[i].get(), validInputSize, stateNumber);
}
}
return true;
}
auto path = ctx->dir_path() + mPath;
FileSource files(path.c_str());
mExecutor.reset(new mtk::NeuronUsdkExecutor(inputTensor.size(), files));
mExecutor->setNumInputs(inputTensor.size() + maskNumber + stateNumber);
mExecutor->setNumOutputs(outputTensor.size() + stateNumber);
mExecutor->initialize();
_loadState(mExecutor.get(), inputTensor.size(), stateNumber);
return true;
}
bool resize(CPUKernelContext* ctx) override {
// TODO: Check Backend type
auto cpuBn = static_cast<CPUBackend*>(ctx->backend());
if (!mAllExecutors.empty()) {
int index = -1;
auto res = computeIndex(ctx->inputs(), ctx->getAttr("allInputShape"), index);
if (!res) {
return false;
}
mExecutor = mAllExecutors[index];
}
auto inputTensor = ctx->inputs();
mInputs.resize(inputTensor.size());
mInputs.clear();
int index = 0;
std::vector<Tensor*> needRelease;
for (int i=0; i<inputTensor.size(); ++i) {
// For Neuropilot will remove all scalar inputs offline, because it may cause crash
if (inputTensor[i]->elementSize() == 1) {
continue;
}
std::pair<const MNN::Tensor *, size_t> inputPair;
inputPair.first = inputTensor[i];
auto& buffer = mExecutor->getInput(index);
inputPair.second = cpuBn->getTensorSize(inputPair.first, true);
mInputs.emplace_back(inputPair);
index++;
}
auto outputTensor = ctx->outputs();
mOutputs.resize(outputTensor.size());
for (int i=0; i<outputTensor.size(); ++i) {
auto& buffer = mExecutor->getOutput(i);
mOutputs[i].first = outputTensor[i];
mOutputs[i].second = cpuBn->getTensorSize(mOutputs[i].first, true);
if (mOutputs[i].second == buffer.sizeBytes) {
outputTensor[i]->buffer().host = (uint8_t*)buffer.buffer;
} else {
// TODO: Support Dynamic mode
bool res = cpuBn->onAcquireBuffer(outputTensor[i], Backend::STATIC);
if (!res) {
return OUT_OF_MEMORY;
}
}
}
return true;
}
bool compute(CPUKernelContext* ctx) override {
auto inputTensor = ctx->inputs();
for (int i=0; i<mInputs.size(); ++i) {
auto& buffer = mExecutor->getInput(i);
if (mInputs[i].first->buffer().host == buffer.buffer) {
continue;
}
if (mInputs[i].second == buffer.sizeBytes) {
::memcpy(buffer.buffer, mInputs[i].first->host<void>(), buffer.sizeBytes);
} else if (mInputs[i].second == buffer.sizeBytes * 2) {
// Float2Half
MNNQuantizeFP16(mInputs[i].first->host<float>(), (int16_t*)buffer.buffer, buffer.sizeBytes / 2);
} else {
MNN_ERROR("For %s, %d input size not math: needed: %ld: input: %ld\n", mPath.c_str(), i, buffer.sizeBytes, mInputs[i].second);
}
}
// If has remove, remove invalid state
auto meta = (KVMeta*)(ctx->backend()->getMetaPtr());
if (nullptr != meta && mStateInput.size() > 0) {
if (meta->remove > 0) {
mStateCurrent-= meta->remove;
if (mMaskFp16) {
auto maskPtr = (__fp16*)mMask.buffer;
for (int i=0; i<meta->remove; ++i) {
maskPtr[i+mStateCurrent] = mMinValue;
}
} else {
auto maskPtr = (float*)mMask.buffer;
for (int i=0; i<meta->remove; ++i) {
maskPtr[i+mStateCurrent] = mMinValue;
}
}
}
}
mExecutor->runInference();
// Update State
if (nullptr != meta && mStateInput.size() > 0) {
auto maskPtr = (float*)mMask.buffer;
if (mMaskFp16) {
auto maskPtr = (__fp16*)mMask.buffer;
for (int i=0; i<meta->add; ++i) {
maskPtr[i+mStateCurrent] = 0.0f;
}
} else {
auto maskPtr = (float*)mMask.buffer;
for (int i=0; i<meta->add; ++i) {
maskPtr[i+mStateCurrent] = 0.0f;
}
}
// Temply use StateOutputs[0] size to compute seq_len
int bytes = mMaskFp16 ? 2 : 4;
int seqLen = mExecutor->getModelOutputSizeBytes(mOutputs.size()) / mStateInput[0].inside / bytes / mStateInput[0].outside;
for (int i=0; i<mStateInput.size(); ++i) {
auto& input = mStateInput[i];
auto& buffer = mExecutor->getOutput(i + mOutputs.size());
for (int y=0; y<input.outside; ++y) {
auto dstOffset = y * input.inside * mStateMaxSize + mStateCurrent * input.inside;
auto srcOffset = y * input.inside * seqLen;
auto dst = (uint8_t*)input.data.buffer + dstOffset * bytes;
auto src = (uint8_t*)buffer.buffer + srcOffset * bytes;
::memcpy(dst, src, meta->add * input.inside * bytes);
}
}
mStateCurrent += meta->add;
}
auto outputTensor = ctx->outputs();
for (int i=0; i<mOutputs.size(); ++i) {
auto& buffer = mExecutor->getOutput(i);
if (mOutputs[i].first->buffer().host == buffer.buffer) {
continue;
}
if (mOutputs[i].second == buffer.sizeBytes) {
::memcpy(mOutputs[i].first->buffer().host, buffer.buffer, buffer.sizeBytes);
} else if (mOutputs[i].second == buffer.sizeBytes * 2) {
MNNDequantizeFP16((int16_t*)buffer.buffer, mOutputs[i].first->host<float>(), buffer.sizeBytes / 2);
} else {
MNN_ERROR("For %s, %d output size not math: needed: %ld: real: %ld\n", mPath.c_str(), i, buffer.sizeBytes, mOutputs[i].second);
}
}
return true;
}
};
} // namespace backend
}
#endif
#endif
void registerNeuroPilot() {
#ifdef MNN_NEUROPILOT_CONVERT_MODE
MNNInsertExtraRuntimeCreator(MNN_CONVERT_NEUROPILOT, new NeuropilotRuntimeCreator, false);
#else
#ifdef MNN_WITH_PLUGIN
plugin::InferShapeKernelRegister::add("MLDA", []() { // NOLINT
return new plugin::shape_inference::PluginShapeRaw; // NOLINT
});
plugin::ComputeKernelRegistry<plugin::backend::PluginExecuteRaw::KernelT>::add("MLDA", []() {
return new plugin::backend::PluginExecuteRaw;
});
#endif
#endif
}
}
source/backend/neuropilot/backend/NeuropilotBackend.hpp
+109
View File
@@ -0,0 +1,109 @@
#ifndef NeuropilotBackend_hpp
#define NeuropilotBackend_hpp
#ifdef MNN_NEUROPILOT_CONVERT_MODE
#include "core/Backend.hpp"
#include "converter/ConvertTflite.hpp"
#include <memory>
#include <map>
namespace MNN {
class NeuropilotRuntime : public Runtime {
public:
~NeuropilotRuntime() {
// Do nothing
}
NeuropilotRuntime(const Backend::Info& info) {
// Do nothing
}
virtual Backend* onCreate(const BackendConfig* config = nullptr, Backend* origin = nullptr) const override;
virtual CompilerType onGetCompilerType() const override {
return Compiler_Origin;
}
virtual bool onSetCachePath(const char* path, int mode) override {
pCachePath = path;
return true;
}
virtual void onGabageCollect(int level) override {
// Do nothing
}
std::string pCachePath;
private:
};
class NeuropilotBackend : public Backend {
public:
NeuropilotBackend(const NeuropilotRuntime* runtime) : Backend(MNN_CONVERT_NEUROPILOT) {
mRuntime = runtime;
}
virtual ~NeuropilotBackend() {
// Do nothing
}
virtual Execution* onCreate(const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, const MNN::Op* op) override;
virtual void onResizeBegin() override;
virtual ErrorCode onResizeEnd() override;
virtual void onExecuteBegin() const override {
// Do nothing
}
virtual void onExecuteEnd() const override {
// Do nothing
}
virtual bool onClearBuffer() override {
return true;
}
virtual MemObj* onAcquire(const Tensor* tensor, StorageType storageType) override;
virtual void onCopyBuffer(const Tensor* srcTensor, const Tensor* dstTensor) const override {
// Do nothing
return;
}
class Creator {
public:
virtual Execution* onCreate(const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs,
const MNN::Op* op, Backend* backend) const = 0;
};
static bool addCreator(OpType t, Creator* c);
virtual const Runtime* getRuntime() override {
return mRuntime;
}
friend class ConvertExecution;
struct ExecuteInfo {
const MNN::Op* op;
std::vector<Tensor*> inputs;
std::vector<Tensor*> outputs;
};
void prepareTensorQuantInfo(const Tensor* tensor, std::unique_ptr<tflite::QuantizationParametersT>&& param);
void setPackTensor(const Tensor* tensor, int packBits = 4);
void setTensorName(const Tensor* tensor, std::string name);
void insertExtraInput(Tensor* tensor);
void insertExtraOutput(Tensor* tensor);
Tensor* getStateMask(int maxLength);
Tensor* getConstTensor(std::string name, std::shared_ptr<Tensor> ref);
private:
// TensorFlow Lite 转换相关的辅助函数
std::unique_ptr<tflite::ModelT> createTensorFlowLiteModel();
void saveTensorFlowLiteModel(std::unique_ptr<tflite::ModelT>& model, const std::string& filePath);
std::map<const Tensor*, int> mTensorIndexMap;
int _createTensorFromMNNTensor(const Tensor* tensor, tflite::SubGraphT* dstGraph, std::vector<std::unique_ptr<tflite::BufferT>>& dstBuffers);
private:
std::shared_ptr<Tensor> mStateMask;
std::map<std::string, std::shared_ptr<Tensor>> mSharedConst;
std::map<const Tensor*, int> mExtraInputs;
std::map<const Tensor*, int> mExtraOutputs;
std::map<int, const Tensor*> mDequantTensor;
std::map<const Tensor*, int> mPackInfo;
std::map<const Tensor*, std::string> mUserTensorName;
const NeuropilotRuntime* mRuntime = nullptr;
std::vector<ExecuteInfo> mInfos;
std::map<const Tensor*, std::unique_ptr<tflite::QuantizationParametersT>> mQuantInfo;
std::map<int, int> mIOIndexMap; // First: MNN's index, Second: Tflite's index
};
};
#endif
#endif
source/backend/neuropilot/converter/AttentionConverter.cpp
+161
View File
@@ -0,0 +1,161 @@
#include <cmath>
#include "AttentionConverter.hpp"
#include "core/TensorUtils.hpp"
#include "backend/NeuropilotBackend.hpp"
namespace MNN {
/**
Q = _Reshape(Q, {batch, seqLength, kvNumHead,group, headDim});
Q = _Transpose(Q, {0, 2, 3, 1, 4});
K = _Reshape(K, {batch, seqLength, kvNumHead, 1, headDim});
K = _Transpose(K, {0, 2, 3, 1, 4});
auto scale = 1.0f / sqrtf(headDim);
K = K * _Scalar<float>(scale);
K.fix(VARP::CONSTANT);
auto QK = _MatMul(Q, K, false, true); // [batch, kvNumHead, group , seq_len, seq_len]
QK = QK + mask;
auto QKPast = _MatMul(Q, cache.pastK, false, true);
QKPast = QKPast + cache.pastMask;
QK = _Concat({QKPast, QK}, -1);
QK = _Softmax(QK, -1);
V = _Reshape(V, {batch, seqLength, kvNumHead, 1, headDim});
V = _Transpose(V, {0, 2, 3, 1, 4});
V.fix(VARP::CONSTANT);
auto totalV = _Concat({cache.pastV, V}, 3);
auto QKV = _MatMul(QK, totalV, false, false);
auto info = QKV->getInfo();
auto O = _Transpose(QKV, {0, 3, 1, 2, 4});
O = _Reshape(O, {batch, seqLength, -1});
O.fix(VARP::CONSTANT);
*/
ConvertTflite::CommandBuffer AttentionConverter::onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) {
auto kvMaxSize = root->pBackend->getRuntime()->hint().kvcacheSizeLimit;
ConvertTflite::CommandBuffer res;
auto Q = inputs[0];
auto K = inputs[1];
auto V = inputs[2];
auto mask = inputs[3];
auto seqLength = Q->length(1);
auto numHead = Q->length(2);
auto headDim = Q->length(3);
auto kvNumHead = K->length(2);
auto batch = Q->length(0);
auto group = numHead / kvNumHead;
auto scale = 1.0f / sqrtf(headDim);
auto attn = op->main_as_AttentionParam();
bool needState = false;
if (nullptr != attn && attn->kv_cache()) {
needState = true;
}
MNN_ASSERT(batch == 1);
if (1 != batch) {
MNN_ERROR("Don't support batch > 1 for mtk npu attention\n");
return res;
}
// Q = _Reshape(Q, {seqLength, kvNumHead,group, headDim});
// Q = _Transpose(Q, {1, 2, 0, 3});
// K = _Reshape(K, {batch, seqLength, kvNumHead, 1, headDim});
// K = _Transpose(K, {1, 2, 0, 3});
Q = root->makeReshape(res, Q, {seqLength, kvNumHead,group, headDim});
Q = root->makeTranspose(res, Q, {1, 2, 0, 3});
Q = root->makeReshape(res, Q, {1, kvNumHead * group, seqLength, headDim});
K = root->makeReshape(res, K, {seqLength, kvNumHead,1, headDim});
K = root->makeTranspose(res, K, {2, 1, 0, 3}); // {1, kvNumHead, seqLength, headDim}
Tensor* pastK = nullptr;
Tensor* pastV = nullptr;
Tensor* stateMask = nullptr;
if (needState) {
stateMask = root->pBackend->getStateMask(kvMaxSize);
// Create pk, pv
std::shared_ptr<Tensor> pastKWrap(Tensor::createDevice<float>({1, kvNumHead, kvMaxSize, headDim}));
pastK = pastKWrap.get();
root->pBackend->insertExtraInput(pastKWrap.get());
res.extraConst.emplace_back(pastKWrap);
std::shared_ptr<Tensor> pastVWrap(Tensor::createDevice<float>({1, kvNumHead, kvMaxSize, headDim}));
pastV = pastVWrap.get();
root->pBackend->insertExtraInput(pastVWrap.get());
res.extraConst.emplace_back(pastVWrap);
}
// K = K * scale
{
std::shared_ptr<Tensor> scaleTensor(Tensor::create<float>({}));
TensorUtils::getDescribe(scaleTensor.get())->usage = Tensor::InsideDescribe::Usage::CONSTANT;
scaleTensor->host<float>()[0] = scale;
res.extraConst.emplace_back(scaleTensor);
K = root->makeBinary(res, K, scaleTensor.get(), tflite::BuiltinOperator_MUL);
if (needState) {
root->pBackend->insertExtraOutput(K);
}
}
// auto QK = _MatMul(Q, K, false, true); // [batch, kvNumHead, group , seq_len, seq_len]
Tensor* QK = nullptr;
{
// Tile firstly
if (1 != group) {
K = root->makeTile(res, K, {1, 1, group, 1});
K = root->makeReshape(res, K, {1, kvNumHead * group, seqLength, headDim});
}
// Matmul
std::shared_ptr<Tensor> qktensor(Tensor::createDevice<float>({1, kvNumHead * group, seqLength, seqLength}));
root->makeMatMul(res, Q, K, false, true, qktensor.get());
QK = qktensor.get();
res.extraConst.emplace_back(qktensor);
}
// QK = QK + mask;
if (seqLength != 1) {
// For decode don't need mask, if add mask will cause crash for neuropilot
QK = root->makeBinary(res, QK, mask, tflite::BuiltinOperator_ADD);
}
if (needState) {
// auto QKPast = _MatMul(Q, pastK, false, true);
// Tile firstly
if (1 != group) {
pastK = root->makeTile(res, pastK, {1, 1, group, 1});
pastK = root->makeReshape(res, pastK, {1, kvNumHead * group, kvMaxSize, headDim});
}
// Matmul
std::shared_ptr<Tensor> qktensor(Tensor::createDevice<float>({1, kvNumHead * group, seqLength, kvMaxSize}));
root->makeMatMul(res, Q, pastK, false, true, qktensor.get());
res.extraConst.emplace_back(qktensor);
auto QKPastMask = root->makeBinary(res, qktensor.get(), stateMask, tflite::BuiltinOperator_ADD);
QK = root->makeConcat(res, {QKPastMask, QK}, 3);
}
// QK = _Softmax(QK, -1);
{
QK = root->makeSoftmax(res, QK);
}
// V = _Reshape(V, {seqLength, kvNumHead, 1, headDim});
// V = _Transpose(V, {1, 2, 0, 3});
V = root->makeReshape(res, V, {seqLength, kvNumHead,1, headDim});
V = root->makeTranspose(res, V, {2, 1, 0, 3}); // 1, kvNumHead, seqLength, headDim
if (needState) {
root->pBackend->insertExtraOutput(V);
V = root->makeConcat(res, {pastV, V}, 2);
}
// auto QKV = _MatMul(QK, V, false, false);
Tensor* QKV = nullptr;
{
// Tile firstly
if (1 != group) {
V = root->makeTile(res, V, {1, 1, group, 1});
V = root->makeReshape(res, V, {1, kvNumHead * group, seqLength + kvMaxSize, headDim});
}
// Matmul
std::shared_ptr<Tensor> qkvtensor(Tensor::createDevice<float>({1, kvNumHead * group, seqLength, headDim}));
root->makeMatMul(res, QK, V, false, false, qkvtensor.get());
QKV = qkvtensor.get();
res.extraConst.emplace_back(qkvtensor);
}
// auto O = _Transpose(QKV, {0, 3, 1, 2, 4});
// O = _Reshape(O, {batch, seqLength, -1});
auto O = root->makeTranspose(res, QKV, {0, 2, 1, 3});
root->makeReshape(res, O, {1, seqLength, -1}, outputs[0]);
return res;
}
};
source/backend/neuropilot/converter/AttentionConverter.hpp
+15
View File
@@ -0,0 +1,15 @@
#ifndef AttentionConverter_hpp
#define AttentionConverter_hpp
#include "ConvertTflite.hpp"
namespace MNN {
class AttentionConverter : public ConvertTflite::Convert {
public:
virtual ConvertTflite::CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) override;
};
};
#endif
source/backend/neuropilot/converter/ConvertExecution.cpp
+30
View File
@@ -0,0 +1,30 @@
#include "ConvertExecution.hpp"
#include "backend/NeuropilotBackend.hpp"
namespace MNN {
ConvertExecution::~ConvertExecution() {
if (nullptr != mHostPtr) {
delete mHostPtr;
}
}
ConvertExecution::ConvertExecution(Backend* bn, const Op* op) : Execution(bn) {
mOp = op;
if (op->main_type() == OpParameter_LayerNorm || op->main_type() == OpParameter_Scale) {
// Copy Op because it may has exteranl
std::unique_ptr<OpT> opt(op->UnPack());
flatbuffers::FlatBufferBuilder builder;
size_t size, offset;
builder.Finish(Op::Pack(builder, opt.get()));
mHostPtr = builder.ReleaseRaw(size, offset);
mOp = flatbuffers::GetRoot<Op>((mHostPtr+offset));
}
}
ErrorCode ConvertExecution::onResize(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs) {
NeuropilotBackend::ExecuteInfo info;
info.op = mOp;
info.inputs = inputs;
info.outputs = outputs;
static_cast<NeuropilotBackend*>(backend())->mInfos.emplace_back(std::move(info));
return NO_ERROR;
}
};
source/backend/neuropilot/converter/ConvertExecution.hpp
+18
View File
@@ -0,0 +1,18 @@
#ifndef ConvertExecution_hpp
#define ConvertExecution_hpp
#include "core/Execution.hpp"
namespace MNN {
class ConvertExecution : public Execution {
public:
ConvertExecution(Backend* bn, const Op* op);
virtual ~ConvertExecution();
virtual ErrorCode onResize(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs) override;
virtual ErrorCode onExecute(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs) override {
return NO_ERROR;
}
private:
const Op* mOp;
uint8_t* mHostPtr = nullptr;
};
};
#endif
source/backend/neuropilot/converter/ConvertTflite.cpp
+377
View File
@@ -0,0 +1,377 @@
#include "core/TensorUtils.hpp"
#include "shape/SizeComputer.hpp"
#include "ConvertTflite.hpp"
#include "ConvolutionTfliteConverter.hpp"
#include "PoolTfliteConverter.hpp"
#include "SingleConvert.hpp"
#include "UnaryTfliteConverter.hpp"
#include "AttentionConverter.hpp"
#include "MTKEXT.hpp"
namespace MNN {
std::shared_ptr<Tensor> ConvertTflite::getIntArrayTensor(std::vector<int> shapes) {
std::shared_ptr<Tensor> shape(Tensor::create<int>({(int)shapes.size()}));
::memcpy(shape->host<int>(), shapes.data(), shapes.size() * sizeof(int));
TensorUtils::getDescribe(shape.get())->usage = Tensor::InsideDescribe::CONSTANT;
return shape;
}
Tensor* ConvertTflite::makeSlice(CommandBuffer& res, Tensor* input, int sta, int size, int axis) {
if (axis < 0) {
axis = axis + input->dimensions();
}
auto shape = input->shape();
shape[axis] = size;
std::shared_ptr<Tensor> output(Tensor::createDevice(shape, input->getType()));
TensorUtils::getDescribe(output.get())->applyQuant = TensorUtils::getDescribe(input)->applyQuant;
TensorUtils::getDescribe(output.get())->quantAttr = TensorUtils::getDescribe(input)->quantAttr;
res.extraConst.emplace_back(output);
std::vector<int> begin(shape.size(), 0);
begin[axis] = sta;
std::vector<int> end = shape;
end[axis] = sta + size;
std::vector<int> stride(shape.size(), 1);
auto beginT = getIntArrayTensor(begin);
auto endT = getIntArrayTensor(end);
auto strideT = getIntArrayTensor(stride);
res.extraConst.emplace_back(beginT);
res.extraConst.emplace_back(endT);
res.extraConst.emplace_back(strideT);
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(tflite::BuiltinOperator_STRIDED_SLICE);
cmd.op->builtin_options.type = tflite::BuiltinOptions_StridedSliceOptions;
cmd.op->builtin_options.value = new tflite::StridedSliceOptionsT;
cmd.inputs = {input, beginT.get(), endT.get(), strideT.get()};
cmd.outputs = {output.get()};
res.commands.emplace_back(std::move(cmd));
return output.get();
}
Tensor* ConvertTflite::makeConcat(CommandBuffer& res, std::vector<Tensor*> inputs, int axis) {
auto i0 = inputs[0];
if (axis < 0) {
axis = axis + i0->dimensions();
}
auto shape = i0->shape();
for (int i=1; i<inputs.size(); ++i) {
shape[axis] += inputs[i]->length(axis);
}
std::shared_ptr<Tensor> output(Tensor::createDevice(shape, i0->getType()));
res.extraConst.emplace_back(output);
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(tflite::BuiltinOperator_CONCATENATION);
cmd.op->builtin_options.type = tflite::BuiltinOptions_ConcatenationOptions;
cmd.op->builtin_options.value = new tflite::ConcatenationOptionsT;
cmd.op->builtin_options.AsConcatenationOptions()->axis = axis;
cmd.inputs = inputs;
cmd.outputs = {output.get()};
res.commands.emplace_back(std::move(cmd));
return output.get();
}
Tensor* ConvertTflite::makeReshape(CommandBuffer& res, Tensor* tensor, std::vector<int> reshapeSize, Tensor* outputUser) {
ConvertTflite::Command cmd;
auto reshapeTensor = ConvertTflite::getIntArrayTensor(reshapeSize);
res.extraConst.emplace_back(reshapeTensor);
if (outputUser == nullptr) {
std::shared_ptr<Tensor> reshapeOutput(Tensor::createDevice(reshapeSize, tensor->getType()));
TensorUtils::getDescribe(reshapeOutput.get())->applyQuant = TensorUtils::getDescribe(tensor)->applyQuant;
TensorUtils::getDescribe(reshapeOutput.get())->quantAttr = TensorUtils::getDescribe(tensor)->quantAttr;
res.extraConst.emplace_back(reshapeOutput);
outputUser = reshapeOutput.get();
}
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(tflite::BuiltinOperator_RESHAPE);
cmd.outputs = {outputUser};
cmd.inputs = {tensor, reshapeTensor.get()};
res.commands.emplace_back(std::move(cmd));
return outputUser;
}
Tensor* ConvertTflite::makeTranspose(CommandBuffer& res, Tensor* tensor, std::vector<int> dims) {
auto dimTensor = ConvertTflite::getIntArrayTensor(dims);
auto tensorShape = getShapeOfTensor(tensor);
std::vector<int> reshapeSize(tensorShape.size());
MNN_ASSERT(dims.size() == tensorShape.size());
for (int i=0; i<dims.size(); ++i) {
reshapeSize[i] = tensorShape[dims[i]];
}
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(tflite::BuiltinOperator_TRANSPOSE);
std::shared_ptr<Tensor> reshapeOutput(Tensor::createDevice(reshapeSize, tensor->getType()));
TensorUtils::getDescribe(reshapeOutput.get())->applyQuant = TensorUtils::getDescribe(tensor)->applyQuant;
TensorUtils::getDescribe(reshapeOutput.get())->quantAttr = TensorUtils::getDescribe(tensor)->quantAttr;
cmd.outputs = {reshapeOutput.get()};
cmd.inputs = {tensor, dimTensor.get()};
auto output = reshapeOutput.get();
res.extraConst.emplace_back(dimTensor);
res.extraConst.emplace_back(reshapeOutput);
res.commands.emplace_back(std::move(cmd));
return output;
}
Tensor* ConvertTflite::makeTile(CommandBuffer& res, Tensor* tensor, std::vector<int> dims) {
auto dimTensor = ConvertTflite::getIntArrayTensor(dims);
auto tensorShape = getShapeOfTensor(tensor);
std::vector<int> reshapeSize(tensorShape.size());
MNN_ASSERT(dims.size() == tensorShape.size());
for (int i=0; i<dims.size(); ++i) {
reshapeSize[i] = tensorShape[i] * dims[i];
}
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(tflite::BuiltinOperator_TILE);
std::shared_ptr<Tensor> reshapeOutput(Tensor::createDevice(reshapeSize, tensor->getType()));
TensorUtils::getDescribe(reshapeOutput.get())->applyQuant = TensorUtils::getDescribe(tensor)->applyQuant;
TensorUtils::getDescribe(reshapeOutput.get())->quantAttr = TensorUtils::getDescribe(tensor)->quantAttr;
cmd.outputs = {reshapeOutput.get()};
cmd.inputs = {tensor, dimTensor.get()};
auto output = reshapeOutput.get();
res.extraConst.emplace_back(dimTensor);
res.extraConst.emplace_back(reshapeOutput);
res.commands.emplace_back(std::move(cmd));
return output;
}
Tensor* ConvertTflite::makeBinary(CommandBuffer& res, Tensor* A, Tensor* B, tflite::BuiltinOperator operation) {
std::shared_ptr<Tensor> tensor(Tensor::createDevice({}, A->getType()));
SizeComputer::computeBroadCastDims({A, B}, {tensor.get()});
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(operation);
cmd.inputs = {A, B};
cmd.outputs = {tensor.get()};
res.extraConst.emplace_back(tensor);
res.commands.emplace_back(std::move(cmd));
return tensor.get();
}
void ConvertTflite::makeMatMul(CommandBuffer& res, Tensor* A, Tensor* B, bool adjA, bool adjB, Tensor* C) {
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(tflite::BuiltinOperator_BATCH_MATMUL);
cmd.op->builtin_options.type = tflite::BuiltinOptions_BatchMatMulOptions;
cmd.op->builtin_options.value = new tflite::BatchMatMulOptionsT;
cmd.op->builtin_options.AsBatchMatMulOptions()->adj_x = adjA;
cmd.op->builtin_options.AsBatchMatMulOptions()->adj_y = adjB;
cmd.inputs = {A, B};
cmd.outputs = {C};
res.commands.emplace_back(std::move(cmd));
}
Tensor* ConvertTflite::makeSoftmax(CommandBuffer& res, Tensor* A) {
std::shared_ptr<Tensor> tensor(Tensor::createDevice(getShapeOfTensor(A), A->getType()));
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = getOpIndex(tflite::BuiltinOperator_SOFTMAX);
cmd.op->builtin_options.type = tflite::BuiltinOptions_SoftmaxOptions;
cmd.op->builtin_options.value = new tflite::SoftmaxOptionsT;
cmd.op->builtin_options.AsSoftmaxOptions()->beta = 1.0f;
cmd.inputs = {A};
cmd.outputs = {tensor.get()};
res.extraConst.emplace_back(tensor);
res.commands.emplace_back(std::move(cmd));
return tensor.get();
}
class ConvertTensorTflite : public ConvertTflite::Convert {
public:
virtual ConvertTflite::CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) override {
auto srcFormat = TensorUtils::getDescribe(inputs[0])->dimensionFormat;
auto dstFormat = TensorUtils::getDescribe(outputs[0])->dimensionFormat;
if (MNN_DATA_FORMAT_NC4HW4 == srcFormat) {
srcFormat = MNN_DATA_FORMAT_NHWC;
}
if (MNN_DATA_FORMAT_NC4HW4 == dstFormat) {
dstFormat = MNN_DATA_FORMAT_NHWC;
}
ConvertTflite::CommandBuffer res;
res.op = op;
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
auto batchChannel = inputs[0]->batch() * inputs[0]->channel();
if (srcFormat == dstFormat || inputs[0]->dimensions() == 2 || batchChannel == inputs[0]->elementSize()) {
// Reshape
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_RESHAPE);
auto shapes = ConvertTflite::getShapeOfTensor(outputs[0]);
auto shape = ConvertTflite::getIntArrayTensor(shapes);
cmd.inputs = {inputs[0], shape.get()};
cmd.outputs = outputs;
res.extraConst.emplace_back(std::move(shape));
} else {
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_TRANSPOSE);
std::vector<int> permutes(inputs[0]->dimensions());
if (srcFormat == MNN_DATA_FORMAT_NHWC) {
permutes[0] = 0;
permutes[1] = inputs[0]->dimensions() - 1;
for (int i=1; i<inputs[0]->dimensions()-1; ++i) {
permutes[i+1] = i;
}
} else {
// NCHW -> NHWC
permutes[0] = 0;
permutes[inputs[0]->dimensions()-1] = 1;
for (int i=1; i<inputs[0]->dimensions()-1; ++i) {
permutes[i] = i + 1;
}
}
auto shape = ConvertTflite::getIntArrayTensor(permutes);
cmd.inputs = {inputs[0], shape.get()};
cmd.outputs = outputs;
res.extraConst.emplace_back(std::move(shape));
}
res.commands.emplace_back(std::move(cmd));
return res;
}
};
ConvertTflite::ConvertTflite() {
{
std::shared_ptr<Convert> single(new SingleConvert);
mConverters.insert(std::make_pair(OpType_Concat, single));
mConverters.insert(std::make_pair(OpType_ReLU, single));
mConverters.insert(std::make_pair(OpType_ReLU6, single));
mConverters.insert(std::make_pair(OpType_Reshape, single));
mConverters.insert(std::make_pair(OpType_Transpose, single));
mConverters.insert(std::make_pair(OpType_Softmax, single));
mConverters.insert(std::make_pair(OpType_BinaryOp, single));
mConverters.insert(std::make_pair(OpType_Eltwise, single));
mConverters.insert(std::make_pair(OpType_Cast, single));
mConverters.insert(std::make_pair(OpType_StridedSlice, single));
mConverters.insert(std::make_pair(OpType_Slice, single));
mConverters.insert(std::make_pair(OpType_GatherV2, single));
mConverters.insert(std::make_pair(OpType_Gather, single));
mConverters.insert(std::make_pair(OpType_Int8ToFloat, single));
mConverters.insert(std::make_pair(OpType_FloatToInt8, single));
}
{
std::shared_ptr<Convert> conv(new ConvolutionTfliteConverter);
mConverters.insert(std::make_pair(OpType_Convolution, conv));
mConverters.insert(std::make_pair(OpType_ConvolutionDepthwise, conv));
}
{
std::shared_ptr<Convert> attn(new AttentionConverter);
mConverters.insert(std::make_pair(OpType_Attention, attn));
}
{
std::shared_ptr<Convert> pool(new PoolTfliteConverter);
mConverters.insert(std::make_pair(OpType_Pooling, pool));
}
{
std::shared_ptr<Convert> pool(new UnaryTfliteConverter);
mConverters.insert(std::make_pair(OpType_UnaryOp, pool));
}
{
std::shared_ptr<Convert> convert(new ConvertTensorTflite);
mConverters.insert(std::make_pair(OpType_ConvertTensor, convert));
mConverters.insert(std::make_pair(OpType_Squeeze, convert));
mConverters.insert(std::make_pair(OpType_Unsqueeze, convert));
mConverters.insert(std::make_pair(OpType_Flatten, convert));
mConverters.insert(std::make_pair(OpType_Identity, convert));
}
{
std::shared_ptr<Convert> convert(new MTKEXT);
mConverters.insert(std::make_pair(OpType_LayerNorm, convert));
}
}
ConvertTflite::~ ConvertTflite() {
// Do nothing
}
int ConvertTflite::getOpIndex(tflite::BuiltinOperator op) {
if (mOperatorCodeIndexMap.find(op) != mOperatorCodeIndexMap.end()) {
return mOperatorCodeIndexMap.find(op)->second;
}
int res = (int)mOperatorCodes.size();
mOperatorCodeIndexMap.insert(std::make_pair(op, res));
std::unique_ptr<tflite::OperatorCodeT> operatorCode(new tflite::OperatorCodeT());
operatorCode->builtin_code = op;
operatorCode->version = 1;
mOperatorCodes.emplace_back(std::move(operatorCode));
return res;
}
int ConvertTflite::getCustomOpIndex(std::string name) {
auto iter = mCustomOpIndex.find(name);
if (iter != mCustomOpIndex.end()) {
return iter->second;
}
int res = (int)mOperatorCodes.size();
std::unique_ptr<tflite::OperatorCodeT> operatorCode(new tflite::OperatorCodeT());
operatorCode->builtin_code = tflite::BuiltinOperator_CUSTOM;
operatorCode->custom_code = name;
operatorCode->version = 1;
mCustomOpIndex.insert(std::make_pair(name, res));
mOperatorCodes.emplace_back(std::move(operatorCode));
return res;
}
ConvertTflite::CommandBuffer ConvertTflite::convert(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs) {
auto iter = mConverters.find(op->type());
if (iter != mConverters.end()) {
return iter->second->onExecute(op, inputs, outputs, this);
}
MNN_ERROR("Don't support op convert: %s\n", EnumNameOpType(op->type()));
CommandBuffer res;
res.op = op;
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT);
cmd.op->opcode_index = getCustomOpIndex("Unknown");
cmd.inputs = inputs;
cmd.outputs = outputs;
res.commands.emplace_back(std::move(cmd));
return res;
}
std::vector<int> ConvertTflite::getShapeOfTensor(const Tensor* tensor) {
auto des = TensorUtils::getDescribe(tensor);
std::vector<int> shape;
if (des->dimensionFormat == MNN_DATA_FORMAT_NC4HW4 && tensor->dimensions() > 2) {
// Use NHWC instead of NC4HW4
shape.emplace_back(tensor->length(0));
for (int i=2; i<tensor->dimensions(); ++i) {
shape.emplace_back(tensor->length(i));
}
shape.emplace_back(tensor->length(1));
} else {
for (int i = 0; i < tensor->dimensions(); ++i) {
shape.push_back(tensor->length(i));
}
}
return shape;
}
tflite::TensorType ConvertTflite::getType(const Tensor* tensor) {
auto des = TensorUtils::getDescribe(tensor);
if (des->applyQuant && des->quantAttr.get() != nullptr) {
if (DataType_DT_INT8 == des->quantAttr->type) {
return tflite::TensorType_INT8;
} else if (DataType_DT_INT16 == des->quantAttr->type) {
return tflite::TensorType_INT16;
}
MNN_ERROR("ConvertTflite Don't support quant type: %d\n", des->quantAttr->type);
return tflite::TensorType_FLOAT32;
}
tflite::TensorType type;
switch (tensor->getType().code) {
case halide_type_float:
type = tflite::TensorType_FLOAT32;
break;
case halide_type_int:
if (tensor->getType().bits == 32) {
type = tflite::TensorType_INT32;
} else if (tensor->getType().bits == 8) {
type = tflite::TensorType_INT8;
}
break;
case halide_type_uint:
if (tensor->getType().bits == 8) {
type = tflite::TensorType_UINT8;
}
break;
default:
type = tflite::TensorType_FLOAT32; // 默认
break;
}
return type;
}
};
source/backend/neuropilot/converter/ConvertTflite.hpp
+59
View File
@@ -0,0 +1,59 @@
#ifndef ConvertTflite_hpp
#define ConvertTflite_hpp
#include <memory>
#include <map>
#include <MNN/Tensor.hpp>
#include "MNN_generated.h"
#include "../../../tools/converter/source/tflite/schema/schema_generated.h"
namespace MNN {
class NeuropilotBackend;
class ConvertTflite {
public:
ConvertTflite();
~ ConvertTflite();
struct Command {
std::vector<Tensor*> inputs;
std::vector<Tensor*> outputs;
std::unique_ptr<tflite::OperatorT> op;
};
struct CommandBuffer {
std::vector<Command> commands;
const Op* op;
std::vector<std::shared_ptr<Tensor>> extraConst;
};
CommandBuffer convert(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs);
class Convert {
public:
Convert() = default;
virtual ~Convert() = default;
virtual CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) = 0;
};
static tflite::TensorType getType(const Tensor* tensor);
static std::vector<int> getShapeOfTensor(const Tensor* tensor);
Tensor* makeReshape(CommandBuffer& res, Tensor* tensor, std::vector<int> shape, Tensor* outputUser = nullptr);
Tensor* makeTranspose(CommandBuffer& res, Tensor* tensor, std::vector<int> dims);
Tensor* makeTile(CommandBuffer& res, Tensor* tensor, std::vector<int> dims);
Tensor* makeBinary(CommandBuffer& res, Tensor* A, Tensor* B, tflite::BuiltinOperator operation);
Tensor* makeSoftmax(CommandBuffer& res, Tensor* tensor);
Tensor* makeConcat(CommandBuffer& res, std::vector<Tensor*> inputs, int axis);
void makeMatMul(CommandBuffer& res, Tensor* A, Tensor* B, bool adjA, bool adB, Tensor* dst);
Tensor* makeSlice(CommandBuffer& res, Tensor* input, int sta, int size, int axis);
std::vector<std::unique_ptr<tflite::OperatorCodeT>> releaseCodes() {
mOperatorCodeIndexMap.clear();
return std::move(mOperatorCodes);
}
int getOpIndex(tflite::BuiltinOperator op);
int getCustomOpIndex(std::string name);
static std::shared_ptr<Tensor> getIntArrayTensor(std::vector<int> shapes);
NeuropilotBackend* pBackend = nullptr;
private:
std::map<tflite::BuiltinOperator, int> mOperatorCodeIndexMap;
std::map<std::string, int> mCustomOpIndex;
std::vector<std::unique_ptr<tflite::OperatorCodeT>> mOperatorCodes;
std::map<OpType, std::shared_ptr<Convert>> mConverters;
};
}
#endif
source/backend/neuropilot/converter/ConvolutionTfliteConverter.cpp
+397
View File
@@ -0,0 +1,397 @@
#include <cmath>
#include "backend/NeuropilotBackend.hpp"
#include "ConvolutionTfliteConverter.hpp"
#include "core/OpCommonUtils.hpp"
#include "core/ConvolutionCommon.hpp"
#include "flatbuffers/flexbuffers.h"
namespace MNN {
template<typename T> void _tranposeWeight(T* dstWeight, const T* originWeight, int group, int kernelSize, int ic, int oc) {
for (int oz=0; oz<oc/group; ++oz) {
for (int k=0; k<kernelSize; ++k) {
for (int iz=0; iz<ic; ++iz) {
dstWeight[oz * kernelSize * ic + k * ic + iz] = originWeight[oz * kernelSize * ic + k + iz * kernelSize];
}
}
}
}
struct ConvConstTensors {
std::shared_ptr<Tensor> weightTensor;
std::shared_ptr<Tensor> biasTensor;
int bits = 0;
};
static ConvConstTensors _getConstTensor(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) {
ConvConstTensors constRes;
if (inputs.size() > 1) {
return constRes;
}
auto conv2d = op->main_as_Convolution2D();
auto common = op->main_as_Convolution2D()->common();
int oc = common->outputCount();
int group = common->group();
int ic = common->inputCount();
if (0 == ic) {
ic = inputs[0]->channel();
}
int kernelX = common->kernelX();
int kernelY = common->kernelY();
std::shared_ptr<Tensor> weightTensor, biasTensor;
bool useQuant = TensorUtils::getDescribe(outputs[0])->applyQuant;
const float* originWeight = nullptr;
const float* originBias = nullptr;
int originWeightSize = 0;
int originBiasSize = 0;
std::shared_ptr<ConvolutionCommon::Int8Common> quanCommon;
if (nullptr != conv2d->quanParameter()) {
bool forceFloat = false;
if (conv2d->quanParameter()->index() != nullptr) {
// The weight is storage as float sparse, but the backend don't support sparse compute, expand it
forceFloat = true;
}
quanCommon = ConvolutionCommon::load(op, nullptr, forceFloat, useQuant);
if (nullptr == quanCommon) {
MNN_ERROR("Memory not Enough, can't extract IDST Convolution: %s \n", op->name()->c_str());
return constRes;
}
// Back to float
originWeight = quanCommon->weightFloat.get();
originWeightSize = quanCommon->weightFloat.size();
} else if (nullptr == conv2d->weight() || nullptr == conv2d->bias()) {
MNN_ERROR("%s has no weight or bias. The model may be benchmark model, please revert the weight/bias firstly\n", op->name()->c_str());
}
if (nullptr == originWeight && nullptr != op->main_as_Convolution2D()->weight()) {
originWeight = op->main_as_Convolution2D()->weight()->data();
originWeightSize = op->main_as_Convolution2D()->weight()->size();
}
if (nullptr == originBias && op->main_as_Convolution2D()->bias()) {
originBias = op->main_as_Convolution2D()->bias()->data();
originBiasSize = op->main_as_Convolution2D()->bias()->size();
}
if (useQuant) {
std::unique_ptr<tflite::QuantizationParametersT> parameters(new tflite::QuantizationParametersT);
biasTensor.reset(Tensor::create<int32_t>({oc}));
TensorUtils::getDescribe(biasTensor.get())->usage = Tensor::InsideDescribe::CONSTANT;
std::unique_ptr<tflite::QuantizationParametersT> biasParameters(new tflite::QuantizationParametersT);
int originBits = quanCommon->originBits;
if (conv2d->symmetricQuan() != nullptr) {
originBits = conv2d->symmetricQuan()->nbits();
}
// Remove too small weight quant scale
auto removeSmallQuantScale = [](int8_t* weight, float* scale, bool async, int alphaSize, int weightSize) {
if (async) {
int kernelCount = alphaSize / 2;
int kernelSize = weightSize / kernelCount;
for (int z=0; z<kernelCount; ++z) {
auto alpha = scale[2 * z + 1];
auto weightZ = weight + z * kernelSize;
if (fabsf(alpha) <= 0.0000000001f) {
scale[2 * z + 1] = 1.0f;
::memset(weightZ, 0, kernelSize);
}
}
} else {
int kernelCount = alphaSize;
int kernelSize = weightSize / kernelCount;
for (int z=0; z<kernelCount; ++z) {
auto alpha = scale[z];
auto weightZ = weight + z * kernelSize;
if (fabsf(alpha) <= 0.0000000001f) {
scale[z] = 1.0f;
::memset(weightZ, 0, kernelSize);
}
}
}
};
constRes.bits = quanCommon->originBits;
if (!quanCommon->canUseInt4) {
weightTensor.reset(Tensor::create<int8_t>({oc/group, kernelY, kernelX, ic}));
auto dstWeight = weightTensor->host<int8_t>();
auto originWeight = quanCommon->weight.get();
TensorUtils::getDescribe(weightTensor.get())->usage = Tensor::InsideDescribe::CONSTANT;
removeSmallQuantScale(originWeight, quanCommon->alpha.get(), quanCommon->asymmetric, quanCommon->alphaSize, quanCommon->weight.size());
_tranposeWeight(dstWeight, originWeight, group, kernelX * kernelY, ic, oc);
if (conv2d->symmetricQuan() != nullptr && conv2d->symmetricQuan()->nbits() <= 4) {
constRes.bits = conv2d->symmetricQuan()->nbits();
}
} else {
weightTensor.reset(Tensor::create<int8_t>({oc/group, kernelY, kernelX, ic}));
auto dstWeight = weightTensor->host<int8_t>();
auto originWeight = quanCommon->weight.get();
TensorUtils::getDescribe(weightTensor.get())->usage = Tensor::InsideDescribe::CONSTANT;
int weightSize = oc/group * ic * kernelX * kernelY;
std::vector<int8_t> tmpWeightStorage;
auto tmpWeight = dstWeight;
if (1 < kernelX * kernelY) {
tmpWeightStorage.resize(weightSize);
tmpWeight = tmpWeightStorage.data();
}
for (int index=0; index<weightSize; ++index) {
uint8_t w_ = originWeight[index / 2];
int truew = index % 2 ? (w_ & 0x0f) : (w_ >> 4);
tmpWeight[index] = (truew - 8);
}
removeSmallQuantScale(tmpWeight, quanCommon->alpha.get(), quanCommon->asymmetric, quanCommon->alphaSize, quanCommon->weight.size());
// static bool gFirst = false;
// if (!gFirst) {
// gFirst = true;
// printf("%s: %d - %d - %d - %d\n", op->name()->c_str(), tmpWeight[0], tmpWeight[1], tmpWeight[2], tmpWeight[3]);
// }
if (tmpWeight != dstWeight) {
_tranposeWeight(dstWeight, tmpWeight, group, kernelX * kernelY, ic, oc);
}
}
if (constRes.bits <= 4) {
root->pBackend->setPackTensor(weightTensor.get(), 4);
}
float maxWeight = (1 << (originBits - 1)) - 1;
float minWeight = -(1 << (originBits - 1));
if (quanCommon->asymmetric) {
auto alpha = quanCommon->alpha.get();
for (int i=0; i<oc; ++i) {
auto scale = alpha[2*i+1];
auto bias = alpha[2*i];
if (fabsf(scale) >= 0.000000001f) {
parameters->scale.emplace_back(scale);
parameters->zero_point.emplace_back((int)(-bias/scale));
parameters->max.emplace_back(maxWeight * scale + bias);
parameters->min.emplace_back(minWeight * scale + bias);
} else {
parameters->scale.emplace_back(0.000000001f);
parameters->zero_point.emplace_back(0);
parameters->max.emplace_back(bias);
parameters->min.emplace_back(bias);
}
}
} else {
for (int i=0; i<oc; ++i) {
auto scale = quanCommon->alpha.get()[i];
if (fabsf(scale) < 0.000000001f) {
scale = 0.00000001f;
}
parameters->scale.emplace_back(scale);
parameters->max.emplace_back(maxWeight * quanCommon->alpha.get()[i]);
parameters->min.emplace_back(minWeight * quanCommon->alpha.get()[i]);
}
parameters->zero_point = std::vector<int64_t>(oc, 0);
}
if (oc == group && group > 1) {
parameters->quantized_dimension = 3;
} else {
parameters->quantized_dimension = 0;
}
auto dstBias = biasTensor->host<int32_t>();
for (int i=0; i<oc; ++i) {
float weightScale = parameters->scale[i];
float inputScale = TensorUtils::getDescribe(inputs[0])->quantAttr->scale;
float biasScale = inputScale * weightScale;
float bias = originBias[i];
if (biasScale > 0.0f) {
dstBias[i] = bias / (biasScale);
biasParameters->scale.emplace_back(biasScale);
} else if (biasScale < 0.0f) {
dstBias[i] = bias / (-biasScale);
biasParameters->scale.emplace_back(-biasScale);
} else {
dstBias[i] = 0;
biasParameters->scale.emplace_back(1.0f);
}
biasParameters->max.emplace_back(bias);
biasParameters->min.emplace_back(bias);
biasParameters->zero_point.emplace_back(0);
}
root->pBackend->prepareTensorQuantInfo(weightTensor.get(), std::move(parameters));
root->pBackend->prepareTensorQuantInfo(biasTensor.get(), std::move(biasParameters));
} else {
weightTensor.reset(Tensor::create<float>({oc/group, kernelY, kernelX, ic}));
TensorUtils::getDescribe(weightTensor.get())->usage = Tensor::InsideDescribe::CONSTANT;
auto dstWeight = weightTensor->host<float>();
_tranposeWeight(dstWeight, originWeight, group, kernelX * kernelY, ic, oc);
biasTensor.reset(Tensor::create<float>({oc}));
TensorUtils::getDescribe(biasTensor.get())->usage = Tensor::InsideDescribe::CONSTANT;
::memcpy(biasTensor->host<void>(), originBias, originBiasSize * sizeof(float));
}
constRes.weightTensor = weightTensor;
constRes.biasTensor = biasTensor;
root->pBackend->setTensorName(weightTensor.get(), op->name()->str() + ".weight");
root->pBackend->setTensorName(biasTensor.get(), op->name()->str() + ".bias");
return constRes;
}
static ConvertTflite::CommandBuffer _makeFullConnect(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root, ConvConstTensors constRes) {
auto conv2d = op->main_as_Convolution2D();
auto common = conv2d->common();
ConvertTflite::CommandBuffer res;
Tensor* fcInput = nullptr;
Tensor* fcOutput = nullptr;
int planeSize = inputs[0]->length(0);
for (int i=2; i<inputs[0]->dimensions(); ++i) {
planeSize *= inputs[0]->length(i);
}
{
ConvertTflite::Command cmd;
std::vector<int> reshapeSize = {1, planeSize, inputs[0]->channel()};
auto reshapeTensor = ConvertTflite::getIntArrayTensor(reshapeSize);
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_RESHAPE);
std::shared_ptr<Tensor> reshapeOutput(Tensor::createDevice<float>(reshapeSize));
TensorUtils::getDescribe(reshapeOutput.get())->applyQuant = TensorUtils::getDescribe(inputs[0])->applyQuant;
TensorUtils::getDescribe(reshapeOutput.get())->quantAttr = TensorUtils::getDescribe(inputs[0])->quantAttr;
cmd.outputs = {reshapeOutput.get()};
cmd.inputs = {inputs[0], reshapeTensor.get()};
fcInput = reshapeOutput.get();
res.extraConst.emplace_back(reshapeTensor);
res.extraConst.emplace_back(reshapeOutput);
res.commands.emplace_back(std::move(cmd));
}
{
// FC
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = root->getCustomOpIndex("MTKEXT_FULLY_CONNECTED");
flexbuffers::Builder builder;
auto start = builder.StartMap();
builder.Int("fused_activation_function", 0);
builder.Bool("keep_num_dims", true);
builder.EndMap(start);
builder.Finish();
int oc = outputs[0]->channel();
int ic = inputs[0]->channel();
cmd.op->custom_options = builder.GetBuffer();
std::shared_ptr<Tensor> fCOutputR(Tensor::createDevice<float>({1, planeSize, outputs[0]->channel()}));
fcOutput = fCOutputR.get();
// Set weight to oc, ic
constRes.weightTensor->buffer().dimensions = 2;
constRes.weightTensor->setLength(0, oc);
constRes.weightTensor->setLength(1, ic);
TensorUtils::setLinearLayout(constRes.weightTensor.get());
cmd.inputs = {fcInput, constRes.weightTensor.get(), constRes.biasTensor.get()};
cmd.outputs = {fcOutput};
TensorUtils::getDescribe(fcOutput)->applyQuant = TensorUtils::getDescribe(outputs[0])->applyQuant;
TensorUtils::getDescribe(fcOutput)->quantAttr = TensorUtils::getDescribe(outputs[0])->quantAttr;
res.commands.emplace_back(std::move(cmd));
res.extraConst.emplace_back(fCOutputR);
res.extraConst.emplace_back(constRes.weightTensor);
res.extraConst.emplace_back(constRes.biasTensor);
}
{
ConvertTflite::Command cmd;
std::vector<int> reshapeSize = ConvertTflite::getShapeOfTensor(outputs[0]);
auto reshapeTensor = ConvertTflite::getIntArrayTensor(reshapeSize);
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_RESHAPE);
std::shared_ptr<Tensor> reshapeOutput(Tensor::createDevice<float>(reshapeSize));
cmd.outputs = {outputs[0]};
cmd.inputs = {fcOutput, reshapeTensor.get()};
res.commands.emplace_back(std::move(cmd));
res.extraConst.emplace_back(reshapeTensor);
}
return res;
}
ConvertTflite::CommandBuffer ConvolutionTfliteConverter::onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) {
auto conv2d = op->main_as_Convolution2D();
auto common = conv2d->common();
bool useQuant = TensorUtils::getDescribe(outputs[0])->applyQuant;
auto constTensors = _getConstTensor(op, inputs, outputs, root);
if (op->type() == OpType_Convolution) {
if (1 == common->kernelX() && 1 == common->kernelY() && 1 == common->strideX() && 1 == common->strideY() && inputs[0]->width() == outputs[0]->width() && inputs[0]->height() == outputs[0]->height() && constTensors.bits == 4) {
// Linear to Convolution
return _makeFullConnect(op, inputs, outputs, root, constTensors);
}
}
ConvertTflite::CommandBuffer res;
res.op = op;
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
switch (op->type()) {
case OpType_Convolution:
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_CONV_2D);
break;
case OpType_ConvolutionDepthwise:
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_DEPTHWISE_CONV_2D);
break;
default:
break;
}
cmd.outputs = outputs;
int oc = common->outputCount();
int group = common->group();
int ic = common->inputCount();
if (0 == ic) {
ic = inputs[0]->channel();
}
int kernelX = common->kernelX();
int kernelY = common->kernelY();
if (op->type() == OpType_Convolution) {
cmd.op->builtin_options.type = tflite::BuiltinOptions_Conv2DOptions;
cmd.op->builtin_options.value = new tflite::Conv2DOptionsT;
auto dstCommon = cmd.op->builtin_options.AsConv2DOptions();
if (useQuant) {
dstCommon->quantized_bias_type = tflite::TensorType_INT32;
}
dstCommon->dilation_h_factor = common->dilateY();
dstCommon->dilation_w_factor = common->dilateX();
dstCommon->stride_h = common->strideY();
dstCommon->stride_w = common->strideX();
// TODO: Fix padding error
switch (common->padMode()) {
case PadMode_VALID:
dstCommon->padding = tflite::Padding_VALID;
break;
case PadMode_SAME:
dstCommon->padding = tflite::Padding_SAME;
break;
default:
dstCommon->padding = tflite::Padding_SAME;
break;
}
if (common->relu()) {
dstCommon->fused_activation_function = tflite::ActivationFunctionType_RELU;
}
if (common->relu6()) {
dstCommon->fused_activation_function = tflite::ActivationFunctionType_RELU6;
}
} else {
cmd.op->builtin_options.type = tflite::BuiltinOptions_DepthwiseConv2DOptions;
cmd.op->builtin_options.value = new tflite::DepthwiseConv2DOptionsT;
auto dstCommon = cmd.op->builtin_options.AsDepthwiseConv2DOptions();
dstCommon->dilation_h_factor = common->dilateY();
dstCommon->dilation_w_factor = common->dilateX();
dstCommon->stride_h = common->strideY();
dstCommon->stride_w = common->strideX();
dstCommon->depth_multiplier = 1;
// TODO: Fix padding error
switch (common->padMode()) {
case PadMode_VALID:
dstCommon->padding = tflite::Padding_VALID;
break;
case PadMode_SAME:
dstCommon->padding = tflite::Padding_SAME;
break;
default:
dstCommon->padding = tflite::Padding_SAME;
break;
}
if (common->relu()) {
dstCommon->fused_activation_function = tflite::ActivationFunctionType_RELU;
}
if (common->relu6()) {
dstCommon->fused_activation_function = tflite::ActivationFunctionType_RELU6;
}
}
if (1 == inputs.size()) {
cmd.inputs = {inputs[0], constTensors.weightTensor.get(), constTensors.biasTensor.get()};
res.extraConst.emplace_back(constTensors.weightTensor);
res.extraConst.emplace_back(constTensors.biasTensor);
}
res.commands.emplace_back(std::move(cmd));
return res;
}
};
source/backend/neuropilot/converter/ConvolutionTfliteConverter.hpp
+12
View File
@@ -0,0 +1,12 @@
#ifndef ConvolutionTfliteConverter_hpp
#define ConvolutionTfliteConverter_hpp
#include "ConvertTflite.hpp"
namespace MNN {
class ConvolutionTfliteConverter : public ConvertTflite::Convert {
public:
virtual ConvertTflite::CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) override;
};
};
#endif
source/backend/neuropilot/converter/MTKEXT.cpp
+65
View File
@@ -0,0 +1,65 @@
#include "MTKEXT.hpp"
#include "core/TensorUtils.hpp"
#include "flatbuffers/flexbuffers.h"
namespace MNN {
ConvertTflite::CommandBuffer MTKEXT::onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) {
ConvertTflite::CommandBuffer res;
if (op->type() == OpType_LayerNorm) {
res.op = op;
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
auto layernorm = op->main_as_LayerNorm();
cmd.inputs = inputs;
cmd.outputs = outputs;
if (!layernorm->useRMSNorm()) {
MNN_ERROR("Don't support not rms norm\n");
}
cmd.op->opcode_index = root->getCustomOpIndex("MTKEXT_RMS_NORMALIZATION");
flexbuffers::Builder builder;
auto start = builder.StartMap();
builder.Float("epsilon", layernorm->epsilon());
builder.EndMap(start);
builder.Finish();
cmd.op->custom_options = builder.GetBuffer();
std::vector<int> axises;
if (nullptr != layernorm->axis()) {
axises.resize(layernorm->axis()->size());
::memcpy(axises.data(), layernorm->axis()->data(), layernorm->axis()->size() * sizeof(int));
for (int i=0; i<axises.size(); ++i) {
if (axises[i] < 0) {
axises[i] = inputs[0]->dimensions() + axises[i];
}
}
}
auto axisTensor = ConvertTflite::getIntArrayTensor(axises);
bool hasGammaBeta = (layernorm->gamma() && layernorm->beta());
int gammasize = 0;
if (hasGammaBeta) {
MNN_ASSERT(layernorm->gamma()->size() == layernorm->beta()->size());
gammasize = layernorm->gamma()->size();
}
std::shared_ptr<MNN::Tensor> gamma;
std::shared_ptr<MNN::Tensor> beta;
if (hasGammaBeta) {
// Use uint8_t to avoid lowp reduce float bytes
gamma.reset(Tensor::create<float>({gammasize}));
beta.reset(Tensor::create<float>({gammasize}));
cmd.inputs = {inputs[0], axisTensor.get(), gamma.get(), beta.get()};
TensorUtils::getDescribe(gamma.get())->usage = Tensor::InsideDescribe::CONSTANT;
TensorUtils::getDescribe(beta.get())->usage = Tensor::InsideDescribe::CONSTANT;
const float* gamma_data = layernorm->gamma()->data();
memcpy(gamma->host<float>(), gamma_data, gammasize * sizeof(float));
const float* beta_data = layernorm->beta()->data();
memcpy(beta->host<float>(), beta_data, gammasize * sizeof(float));
res.extraConst.emplace_back(gamma);
res.extraConst.emplace_back(beta);
} else {
cmd.inputs = {inputs[0], axisTensor.get()};
}
res.extraConst.emplace_back(axisTensor);
res.commands.emplace_back(std::move(cmd));
}
return res;
}
};
source/backend/neuropilot/converter/MTKEXT.hpp
+14
View File
@@ -0,0 +1,14 @@
#ifndef MTKEXT_hpp
#define MTKEXT_hpp
#include "ConvertTflite.hpp"
namespace MNN {
class MTKEXT : public ConvertTflite::Convert {
public:
virtual ConvertTflite::CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) override;
};
};
#endif
source/backend/neuropilot/converter/OptimizeCommandBuffer.cpp
+125
View File
@@ -0,0 +1,125 @@
#include "core/TensorUtils.hpp"
#include "OptimizeCommandBuffer.hpp"
namespace MNN {
OptimizeCommandBuffer::OptimizeCommandBuffer(ConvertTflite* root) {
mRoot = root;
}
OptimizeCommandBuffer::~OptimizeCommandBuffer() {
// Do nothing
}
static void _swapDequantAndReshape(ConvertTflite::CommandBuffer& cmdBuffer, ConvertTflite* root) {
auto reshapeCode = root->getOpIndex(tflite::BuiltinOperator_RESHAPE);
auto dequantCode = root->getOpIndex(tflite::BuiltinOperator_DEQUANTIZE);
std::vector<int> reshapeIndexes;
std::map<Tensor*, int> reshapeOutput;
std::map<Tensor*, int> dequantOutput;
for (int i=0; i<cmdBuffer.commands.size(); ++i) {
if (nullptr == cmdBuffer.commands[i].op.get()) {
continue;
}
if (cmdBuffer.commands[i].op->opcode_index == reshapeCode) {
reshapeIndexes.emplace_back(i);
if (TensorUtils::getDescribe(cmdBuffer.commands[i].outputs[0])->usage != Tensor::InsideDescribe::OUTPUT) {
reshapeOutput.insert(std::make_pair(cmdBuffer.commands[i].outputs[0], i));
}
continue;
}
if (cmdBuffer.commands[i].op->opcode_index == dequantCode) {
auto out = cmdBuffer.commands[i].outputs[0];
auto des = TensorUtils::getDescribe(out);
if (des->usage != Tensor::InsideDescribe::OUTPUT && des->useCount == 1) {
dequantOutput.insert(std::make_pair(cmdBuffer.commands[i].outputs[0], i));
}
continue;
}
}
// Swap Dequant And Reshape
for (int i=0; i<reshapeIndexes.size(); ++i) {
auto& curCmd = cmdBuffer.commands[reshapeIndexes[i]];
auto iter = dequantOutput.find(curCmd.inputs[0]);
if (iter == dequantOutput.end()) {
continue;
}
auto& deqCmd = cmdBuffer.commands[iter->second];
// C (DEQ)->A (RESHAPE)->B ---> C(RESHAPE)->A (DEQ)->B
auto A = curCmd.inputs[0];
auto B = curCmd.outputs[0];
auto C = deqCmd.inputs[0];
deqCmd.inputs[0] = A;
deqCmd.outputs[0] = B;
curCmd.inputs[0] = C;
curCmd.outputs[0] = A;
TensorUtils::getDescribe(A)->quantAttr = TensorUtils::getDescribe(C)->quantAttr;
TensorUtils::getDescribe(A)->applyQuant = TensorUtils::getDescribe(C)->applyQuant;
reshapeIndexes[i] = iter->second;
std::swap(cmdBuffer.commands[reshapeIndexes[i]], cmdBuffer.commands[iter->second]);
}
}
static void _removeDupReshape(ConvertTflite::CommandBuffer& cmdBuffer, ConvertTflite* root) {
auto reshapeCode = root->getOpIndex(tflite::BuiltinOperator_RESHAPE);
// Find reshape
std::vector<int> reshapeIndexes;
std::map<Tensor*, int> reshapeOutput;
for (int i=0; i<cmdBuffer.commands.size(); ++i) {
if (nullptr == cmdBuffer.commands[i].op.get()) {
continue;
}
if (cmdBuffer.commands[i].op->opcode_index == reshapeCode) {
reshapeIndexes.emplace_back(i);
if (TensorUtils::getDescribe(cmdBuffer.commands[i].outputs[0])->usage != Tensor::InsideDescribe::OUTPUT) {
reshapeOutput.insert(std::make_pair(cmdBuffer.commands[i].outputs[0], i));
}
continue;
}
}
bool change;
do {
change = false;
for (int i=1; i<reshapeIndexes.size(); ++i) {
auto& curCmd = cmdBuffer.commands[reshapeIndexes[i]];
if (curCmd.op.get() == nullptr) {
continue;
}
auto iter = reshapeOutput.find(curCmd.inputs[0]);
if (iter != reshapeOutput.end()) {
auto& removeCmd = cmdBuffer.commands[iter->second];
curCmd.inputs[0] = removeCmd.inputs[0];
change = true;
// Change input from
TensorUtils::getDescribe(iter->first)->useCount--;
if (TensorUtils::getDescribe(iter->first)->useCount <= 0) {
removeCmd.op.reset();
}
}
}
} while (change);
}
static void _computeRefCount(ConvertTflite::CommandBuffer& cmdBuffer) {
// Compute RefCount
for (int i=0; i<cmdBuffer.commands.size(); ++i) {
if (cmdBuffer.commands[i].op.get() == nullptr) {
continue;
}
for (auto t : cmdBuffer.commands[i].inputs) {
TensorUtils::getDescribe(t)->useCount = 0;
}
for (auto t : cmdBuffer.commands[i].outputs) {
TensorUtils::getDescribe(t)->useCount = 0;
}
}
for (int i=0; i<cmdBuffer.commands.size(); ++i) {
for (auto t : cmdBuffer.commands[i].inputs) {
TensorUtils::getDescribe(t)->useCount++;
}
}
}
ConvertTflite::CommandBuffer OptimizeCommandBuffer::reduce(ConvertTflite::CommandBuffer&& cmdBuffer) {
// _computeRefCount(cmdBuffer);
// _swapDequantAndReshape(cmdBuffer, mRoot);
_computeRefCount(cmdBuffer);
_removeDupReshape(cmdBuffer, mRoot);
return std::move(cmdBuffer);
}
};
source/backend/neuropilot/converter/OptimizeCommandBuffer.hpp
+14
View File
@@ -0,0 +1,14 @@
#ifndef OptimizeCommandBuffer_hpp
#define OptimizeCommandBuffer_hpp
#include "ConvertTflite.hpp"
namespace MNN {
class OptimizeCommandBuffer {
public:
OptimizeCommandBuffer(ConvertTflite* root);
~ OptimizeCommandBuffer();
ConvertTflite::CommandBuffer reduce(ConvertTflite::CommandBuffer&& cmdBuffer);
private:
ConvertTflite* mRoot;
};
};
#endif
source/backend/neuropilot/converter/PoolTfliteConverter.cpp
+47
View File
@@ -0,0 +1,47 @@
#include "PoolTfliteConverter.hpp"
namespace MNN {
ConvertTflite::CommandBuffer PoolTfliteConverter::onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) {
ConvertTflite::CommandBuffer res;
res.op = op;
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
auto pool = op->main_as_Pool();
auto type = pool->type();
if (type == PoolType_AVEPOOL) {
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_AVERAGE_POOL_2D);
}
if (type == PoolType_MAXPOOL) {
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_MAX_POOL_2D);
}
cmd.op->builtin_options.type = tflite::BuiltinOptions_Pool2DOptions;
cmd.op->builtin_options.value = new tflite::Pool2DOptionsT;
if (pool->isGlobal()) {
cmd.op->builtin_options.AsPool2DOptions()->stride_h = 1;
cmd.op->builtin_options.AsPool2DOptions()->stride_w = 1;
cmd.op->builtin_options.AsPool2DOptions()->filter_height = inputs[0]->height();
cmd.op->builtin_options.AsPool2DOptions()->filter_width = inputs[0]->width();
} else {
cmd.op->builtin_options.AsPool2DOptions()->stride_h = pool->strideY();
cmd.op->builtin_options.AsPool2DOptions()->stride_w = pool->strideX();
cmd.op->builtin_options.AsPool2DOptions()->filter_height = pool->kernelY();
cmd.op->builtin_options.AsPool2DOptions()->filter_width = pool->kernelX();
}
// TODO: Add extra padding for pads
cmd.op->builtin_options.AsPool2DOptions()->padding = tflite::Padding_SAME;
switch (pool->padType()) {
case PoolPadType_VALID:
cmd.op->builtin_options.AsPool2DOptions()->padding = tflite::Padding_VALID;
break;
default:
break;
}
if (pool->isGlobal()) {
cmd.op->builtin_options.AsPool2DOptions()->padding = tflite::Padding_VALID;
}
cmd.inputs = inputs;
cmd.outputs = outputs;
res.commands.emplace_back(std::move(cmd));
return res;
}
};
source/backend/neuropilot/converter/PoolTfliteConverter.hpp
+14
View File
@@ -0,0 +1,14 @@
#ifndef PoolTfliteConverter_hpp
#define PoolTfliteConverter_hpp
#include "ConvertTflite.hpp"
namespace MNN {
class PoolTfliteConverter : public ConvertTflite::Convert {
public:
virtual ConvertTflite::CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) override;
};
};
#endif
source/backend/neuropilot/converter/SingleConvert.cpp
+229
View File
@@ -0,0 +1,229 @@
#include "SingleConvert.hpp"
#include "core/TensorUtils.hpp"
namespace MNN {
static tflite::BuiltinOperator getBinaryOpTFLiteType(int mnnBinaryOpType) {
switch (mnnBinaryOpType) {
case BinaryOpOperation_ADD:
return tflite::BuiltinOperator_ADD;
case BinaryOpOperation_SUB:
return tflite::BuiltinOperator_SUB;
case BinaryOpOperation_MUL:
return tflite::BuiltinOperator_MUL;
case BinaryOpOperation_DIV:
case BinaryOpOperation_REALDIV:
return tflite::BuiltinOperator_DIV;
case BinaryOpOperation_MINIMUM:
return tflite::BuiltinOperator_MINIMUM;
case BinaryOpOperation_MAXIMUM:
return tflite::BuiltinOperator_MAXIMUM;
case BinaryOpOperation_EQUAL:
return tflite::BuiltinOperator_EQUAL;
case BinaryOpOperation_LESS:
return tflite::BuiltinOperator_LESS;
case BinaryOpOperation_LESS_EQUAL:
return tflite::BuiltinOperator_LESS_EQUAL;
case BinaryOpOperation_GREATER:
return tflite::BuiltinOperator_GREATER;
case BinaryOpOperation_GREATER_EQUAL:
return tflite::BuiltinOperator_GREATER_EQUAL;
case BinaryOpOperation_NOTEQUAL:
return tflite::BuiltinOperator_NOT_EQUAL;
case BinaryOpOperation_POW:
return tflite::BuiltinOperator_POW;
case BinaryOpOperation_FLOORDIV:
return tflite::BuiltinOperator_FLOOR_DIV;
default:
MNN_PRINT("Warning: Unsupported MNN BinaryOpOperation %d, using ADD\n", mnnBinaryOpType);
return tflite::BuiltinOperator_ADD;
}
}
static tflite::BuiltinOperator _getEltwiseOp(const MNN::Op* op) {
auto elt = op->main_as_Eltwise();
switch (elt->type()) {
case EltwiseType_SUM:
return tflite::BuiltinOperator_ADD;
case EltwiseType_SUB:
return tflite::BuiltinOperator_SUB;
case EltwiseType_PROD:
return tflite::BuiltinOperator_MUL;
default:
break;
}
return tflite::BuiltinOperator_ADD;
}
static tflite::BuiltinOperator _mapMNNOpToTFLiteOp(const MNN::Op* op) {
auto mnnOpType = op->type();
switch (mnnOpType) {
case OpType_FloatToInt8:
return tflite::BuiltinOperator_QUANTIZE;
case OpType_Int8ToFloat:
return tflite::BuiltinOperator_DEQUANTIZE;
case OpType_StridedSlice:
return tflite::BuiltinOperator_STRIDED_SLICE;
case OpType_Gather:
case OpType_GatherV2:
return tflite::BuiltinOperator_GATHER;
case OpType_Cast:
return tflite::BuiltinOperator_CAST;
case OpType_ReLU:
return tflite::BuiltinOperator_RELU;
case OpType_ReLU6:
return tflite::BuiltinOperator_RELU6;
case OpType_Softmax:
return tflite::BuiltinOperator_SOFTMAX;
case OpType_Slice:
return tflite::BuiltinOperator_SPLIT;
case OpType_Concat:
return tflite::BuiltinOperator_CONCATENATION;
case OpType_Reshape:
return tflite::BuiltinOperator_RESHAPE;
case OpType_Transpose:
return tflite::BuiltinOperator_TRANSPOSE;
case OpType_BinaryOp:
return getBinaryOpTFLiteType(op->main_as_BinaryOp()->opType());
case OpType_Eltwise:
return _getEltwiseOp(op);
default:
MNN_PRINT("Warning: Unsupported MNN OpType %d, using CUSTOM operator\n", static_cast<int>(mnnOpType));
return tflite::BuiltinOperator_CUSTOM;
}
return tflite::BuiltinOperator_CUSTOM;
}
ConvertTflite::CommandBuffer SingleConvert::onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) {
ConvertTflite::CommandBuffer res;
res.op = op;
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT);
auto opcode = _mapMNNOpToTFLiteOp(op);
cmd.op->opcode_index = root->getOpIndex(opcode);
cmd.inputs = inputs;
cmd.outputs = outputs;
if (op->type() == OpType_Softmax) {
cmd.op->builtin_options.type = tflite::BuiltinOptions_SoftmaxOptions;
cmd.op->builtin_options.value = new tflite::SoftmaxOptionsT;
cmd.op->builtin_options.AsSoftmaxOptions()->beta = 1.0f;
} else if (op->type() == OpType_Slice) {
cmd.op->builtin_options.type = tflite::BuiltinOptions_SplitOptions;
cmd.op->builtin_options.value = new tflite::SplitOptionsT;
cmd.op->builtin_options.AsSplitOptions()->num_splits = (int)cmd.outputs.size();
std::vector<int> axis = {op->main_as_Slice()->axis()};
if (TensorUtils::getDescribe(inputs[0])->dimensionFormat == MNN_DATA_FORMAT_NC4HW4) {
// Need transform
auto dimensions = inputs[0]->dimensions();
if (dimensions >= 3) {
if (axis[0] == 1) {
axis[0] = dimensions - 1;
} else if (axis[0] > 1) {
axis[0] = axis[0] - 1;
}
}
}
auto axisTensor = ConvertTflite::getIntArrayTensor(axis);
cmd.inputs = {axisTensor.get(), inputs[0]};
res.extraConst.emplace_back(axisTensor);
} else if (op->type() == OpType_Concat) {
cmd.op->builtin_options.type = tflite::BuiltinOptions_ConcatenationOptions;
cmd.op->builtin_options.value = new tflite::ConcatenationOptionsT;
auto tfliteConcatOption = cmd.op->builtin_options.AsConcatenationOptions();
tfliteConcatOption->axis = op->main_as_Axis()->axis();
} else if (op->type() == OpType_Cast) {
cmd.op->builtin_options.type = tflite::BuiltinOptions_CastOptions;
cmd.op->builtin_options.value = new tflite::CastOptionsT;
auto cast = cmd.op->builtin_options.AsCastOptions();
cast->in_data_type = ConvertTflite::getType(inputs[0]);
cast->out_data_type = ConvertTflite::getType(outputs[0]);
} else if (op->type() == OpType_StridedSlice) {
cmd.op->builtin_options.type = tflite::BuiltinOptions_StridedSliceOptions;
cmd.op->builtin_options.value = new tflite::StridedSliceOptionsT;
auto src = op->main_as_StridedSliceParam();
auto dst = cmd.op->builtin_options.AsStridedSliceOptions();
dst->begin_mask = src->beginMask();
dst->end_mask = src->endMask();
dst->ellipsis_mask = src->ellipsisMask();
dst->new_axis_mask = src->newAxisMask();
dst->shrink_axis_mask = src->shrinkAxisMask();
if (src->fromType() == 1) {
// Change inputs
std::vector<int> begin(inputs[0]->dimensions(), 0);
std::vector<int> end(inputs[0]->dimensions());
for (int i=0; i<inputs[0]->dimensions(); ++i) {
end[i] = inputs[0]->length(i);
}
std::vector<int> stride(inputs[0]->dimensions(), 1);
auto beginT = inputs[1];
auto validSize = beginT->elementSize();
auto endT = inputs[2];
auto axisT = inputs[3];
auto stepT = inputs[4];
for (int i=0; i<validSize; ++i) {
auto axis = axisT->host<int>()[i];
if (axis < 0) {
axis += inputs[0]->dimensions();
}
begin[axis] = beginT->host<int>()[i];
end[axis] = endT->host<int>()[i];
stride[axis] = stepT->host<int>()[i];
}
auto newBegin = ConvertTflite::getIntArrayTensor(begin);
auto newEnd = ConvertTflite::getIntArrayTensor(end);
auto newStride = ConvertTflite::getIntArrayTensor(stride);
cmd.inputs = {inputs[0], newBegin.get(), newEnd.get(), newStride.get()};
res.extraConst.emplace_back(newBegin);
res.extraConst.emplace_back(newEnd);
res.extraConst.emplace_back(newStride);
}
} else if (op->type() == OpType_Reshape) {
std::vector<int> shape = outputs[0]->shape();
auto shapeT = ConvertTflite::getIntArrayTensor(shape);
cmd.inputs = {inputs[0], shapeT.get()};
res.extraConst.emplace_back(shapeT);
} else if (op->type() == OpType_Gather || op->type() == OpType_GatherV2) {
int axis = 0;
if (inputs.size() == 3) {
auto axis_tensor = inputs[2];
axis = axis_tensor->host<int32_t>()[0];
}
if (op->main_type() == OpParameter_Axis) {
axis = op->main_as_Axis()->axis();
}
if (inputs[0]->size() == outputs[0]->size()) {
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_RESHAPE);
std::vector<int> shape = outputs[0]->shape();
auto shapeT = ConvertTflite::getIntArrayTensor(shape);
cmd.inputs = {inputs[0], shapeT.get()};
res.extraConst.emplace_back(shapeT);
} else if(TensorUtils::getDescribe(inputs[1])->usage == Tensor::InsideDescribe::CONSTANT && inputs[1]->elementSize() == 1) {
// Turn to Slice
cmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_STRIDED_SLICE);
cmd.op->builtin_options.type = tflite::BuiltinOptions_StridedSliceOptions;
cmd.op->builtin_options.value = new tflite::StridedSliceOptionsT;
std::vector<int> begin(inputs[0]->dimensions(), 0);
begin[axis] = inputs[1]->host<int>()[0];
std::vector<int> end(inputs[0]->dimensions());
for (int i=0; i<inputs[0]->dimensions(); ++i) {
end[i] = inputs[0]->length(i);
}
end[axis] = begin[axis] + 1;
cmd.op->builtin_options.AsStridedSliceOptions()->shrink_axis_mask = 1 << axis;
std::vector<int> stride(inputs[0]->dimensions(), 1);
auto newBegin = ConvertTflite::getIntArrayTensor(begin);
auto newEnd = ConvertTflite::getIntArrayTensor(end);
auto newStride = ConvertTflite::getIntArrayTensor(stride);
cmd.inputs = {inputs[0], newBegin.get(), newEnd.get(), newStride.get()};
res.extraConst.emplace_back(newBegin);
res.extraConst.emplace_back(newEnd);
res.extraConst.emplace_back(newStride);
} else {
cmd.op->builtin_options.type = tflite::BuiltinOptions_GatherOptions;
cmd.op->builtin_options.value = new tflite::GatherOptionsT;
auto dst = cmd.op->builtin_options.AsGatherOptions();
dst->axis = axis;
cmd.inputs = {inputs[0], inputs[1]};
}
}
res.commands.emplace_back(std::move(cmd));
return res;
}
};
source/backend/neuropilot/converter/SingleConvert.hpp
+13
View File
@@ -0,0 +1,13 @@
#ifndef SingleConvert_hpp
#define SingleConvert_hpp
#include "ConvertTflite.hpp"
namespace MNN {
class SingleConvert : public ConvertTflite::Convert {
public:
virtual ConvertTflite::CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) override;
};
}
#endif
source/backend/neuropilot/converter/UnaryTfliteConverter.cpp
+79
View File
@@ -0,0 +1,79 @@
#include "UnaryTfliteConverter.hpp"
#include "core/TensorUtils.hpp"
namespace MNN {
static tflite::BuiltinOperator _mapUnary(UnaryOpOperation src) {
switch (src) {
case UnaryOpOperation_ABS:
return tflite::BuiltinOperator_ABS;
case UnaryOpOperation_COS:
return tflite::BuiltinOperator_COS;
case UnaryOpOperation_EXP:
return tflite::BuiltinOperator_EXP;
case UnaryOpOperation_LOG:
return tflite::BuiltinOperator_LOG;
case UnaryOpOperation_NEG:
return tflite::BuiltinOperator_NEG;
case UnaryOpOperation_ROUND:
return tflite::BuiltinOperator_ROUND;
case UnaryOpOperation_SIN:
return tflite::BuiltinOperator_SIN;
case UnaryOpOperation_SIGMOID:
return tflite::BuiltinOperator_LOGISTIC;
case UnaryOpOperation_SQRT:
return tflite::BuiltinOperator_SQRT;
case UnaryOpOperation_SQUARE:
return tflite::BuiltinOperator_SQUARE;
case UnaryOpOperation_TANH:
return tflite::BuiltinOperator_TANH;
default:
break;
}
return tflite::BuiltinOperator_CUSTOM;
}
ConvertTflite::CommandBuffer UnaryTfliteConverter::onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) {
ConvertTflite::CommandBuffer res;
res.op = op;
auto param = op->main_as_UnaryOp();
auto code = _mapUnary(param->opType());
if (code != tflite::BuiltinOperator_CUSTOM) {
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = root->getOpIndex(code);
cmd.inputs = inputs;
cmd.outputs = outputs;
res.commands.emplace_back(std::move(cmd));
return res;
}
if (param->opType() == UnaryOpOperation_SILU) {
// sigmoid + mul
std::shared_ptr<Tensor> tensor(new Tensor(inputs[0], inputs[0]->getDimensionType(), false));
TensorUtils::getDescribe(tensor.get())->dimensionFormat = TensorUtils::getDescribe(inputs[0])->dimensionFormat;
// Sigmoid
ConvertTflite::Command sigmoidcmd;
sigmoidcmd.op.reset(new tflite::OperatorT());
sigmoidcmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_LOGISTIC);
sigmoidcmd.inputs = inputs;
sigmoidcmd.outputs = {tensor.get()};
res.commands.emplace_back(std::move(sigmoidcmd));
// Mul
ConvertTflite::Command mulcmd;
mulcmd.op.reset(new tflite::OperatorT());
mulcmd.op->opcode_index = root->getOpIndex(tflite::BuiltinOperator_MUL);
mulcmd.inputs = {inputs[0], tensor.get()};
mulcmd.outputs = outputs;
res.commands.emplace_back(std::move(mulcmd));
res.extraConst.emplace_back(tensor);
return res;
}
MNN_ERROR("Don't support convert %s unary to tflite\n", EnumNameUnaryOpOperation(param->opType()));
ConvertTflite::Command cmd;
cmd.op.reset(new tflite::OperatorT());
cmd.op->opcode_index = root->getCustomOpIndex("UNKNOWN");
cmd.inputs = inputs;
cmd.outputs = outputs;
res.commands.emplace_back(std::move(cmd));
return res;
}
};
source/backend/neuropilot/converter/UnaryTfliteConverter.hpp
+14
View File
@@ -0,0 +1,14 @@
#ifndef UnaryTfliteConverter_hpp
#define UnaryTfliteConverter_hpp
#include "ConvertTflite.hpp"
namespace MNN {
class UnaryTfliteConverter : public ConvertTflite::Convert {
public:
virtual ConvertTflite::CommandBuffer onExecute(const Op* op, const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs, ConvertTflite* root) override;
};
};
#endif
source/backend/neuropilot/mtk/backend/api/neuron/DLAMuxer.h
Executable
+297
View File
@@ -0,0 +1,297 @@
/**
* \file
* Neuron DLA Muxer API
* ---
* Neuron DLA Muxer provides APIs to create runtime NeuronDLAMuxer for parsing packed DLA files, and do
* inference with specified DLA in the pool.
* \n The Runtime user should include this header to use Neuron DLA Muxer API.
*/
#pragma once
#if __has_include("Types.h")
#include "Types.h"
#else
#include "neuron/api/Types.h"
#endif
#include <stddef.h>
#include <stdint.h>
#include <sys/cdefs.h>
__BEGIN_DECLS
/**
* Create a NeuronDLAMuxer based on the setting specified in options and the packed DLAs file
* generated by dla-packer. The address of the created instance will be passed back in *dlaMuxer.
* @param pathToDLB The path to the bundled DLA file.
* @param options The environment options for the created NeuronDLAMuxer.
* @param dlaMuxer DLAMuxer provides API for run a compiled network with a bundled DLA (.dlb) file.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_createWithOptions(const char* pathToDLB, const char* options, void** dlaMuxer);
/**
* Get number of DLAs in the dlaMuxer.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param dlaCount The total number of DLAs in the dlaMuxer.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getDLACount(void* dlaMuxer, size_t* dlaCount);
/**
* Get the index of DLA specified by input name, and return it by dlaIndex.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param name A null-terminated string which indicates the name of target DLA in the dlaMuxer.
* @param dlaIndex The pointer to store the returned index for the target DLA.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getIndexByName(void* dlaMuxer, const char* name, size_t* dlaIndex);
/**
* Do inference with specified DLA set with selectDLA API.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_inference(void* dlaMuxer);
/**
* Set the memory buffer for all the extracted static tensors in the network of the selected DLA.
* If the binary file to the extracted static data is packed into the .dlb file by dla-packer at
* bundle time, DLAMuxer will automatically allocate a static data buffer from the bundled binary
* file, and set all the extracted static tensor from the buffer. If all the parameters (address,
* size, and file descriptor in attribute) to this API are valid, DLAMuxer prefers to use the
* provided buffer instead of the DLAMuxer-allocated buffer (if existed).
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param staticDataBuffer The extracted static data buffer.
* @param length The extracted static data buffer size.
* @param attribute The buffer attribute for the set buffer.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_setStaticData(void* dlaMuxer, const void* staticDataBuffer, size_t length,
BufferAttribute attribute);
/**
* Select a specific DLA in the dlaMuxer by a given index. The selected DLA will act as the
* operation target for later API calls.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param index The DLA index to select.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_selectDLA(void* dlaMuxer, size_t index);
/**
* Get the size of DLA name for the selected DLA.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param size The size of the DLA name.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getDLANameSize(void* dlaMuxer, size_t* size);
/**
* Get the file name for the selected DLA.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param name The destination buffer to store the output DLA name.
* @param size The size of the destination buffer.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getDLAName(void* dlaMuxer, char* name, size_t size);
/**
* Set the memory buffer for the tensor which hold the specified input handle in the original
* network of the selected DLA. If there are multiple inputs, each of them have to be set.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param buffer The input buffer.
* @param length The input buffer size.
* @param attribute The buffer attribute for setting ION.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_setInput(void* dlaMuxer, uint64_t handle, const void* buffer, size_t length,
BufferAttribute attribute);
/**
* Set the memory buffer and offset for the tensor which hold the specified input handle in the
* original network for the selected DLA. If there are multiple inputs, each of them have to be set.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param buffer The input buffer.
* @param length The input buffer size.
* @param attribute The buffer attribute for setting ION.
* @param offset The offset for ION buffer.
* @param offset Reading ION buffer from start addr + offset.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_setOffsetedInput(void* dlaMuxer, uint64_t handle, const void* buffer,
size_t length, BufferAttribute attribute, size_t offset);
/**
* Set the memory buffer for the tensor which hold the specified output handle in the original
* network for the selected DLA. If there are multiple outputs, each of them have to be set.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param buffer The output buffer.
* @param length The output buffer size.
* @param attribute The buffer attribute for setting ION.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_setOutput(void* dlaMuxer, uint64_t handle, void* buffer, size_t length,
BufferAttribute attribute);
/**
* Set the memory buffer and offset for the tensor which hold the specified output handle in the
* original network for the selected DLA. If there are multiple outputs, each of them have to be
* set.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param buffer The output buffer.
* @param length The output buffer size.
* @param attribute The buffer attribute for setting ION.
* @param offset Writing ION buffer from start addr + offset.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_setOffsetedOutput(void* dlaMuxer, uint64_t handle, void* buffer, size_t length,
BufferAttribute attribute, size_t offset);
/**
* Get metadata info in the selected DLA in the dlaMuxer, which is provided through compiler option
* --dla-metadata.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param key The key for the target data
* @param size The size of the target data. If there is no corresponding metadata, size is 0.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getMetadataInfo(void* dlaMuxer, const char* key, size_t* size);
/**
* Get metadata in the selected DLA in the dlaMuxer, which is provided through compiler option
* --dla-metadata.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param key The key for the target data
* @param data The destination data buffer.
* @param size The size to read from metadata.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getMetadata(void* dlaMuxer, const char* key, char* data, size_t size);
/**
* Set the QoS configuration for the selected Neuron Runtime.
* If qosOption.profiledQoSData is not nullptr, the selected Neuron Runtime would use it as the
* profiled QoS data.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param qosOption The option for QoS configuration.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_setQoSOption(void* dlaMuxer, const QoSOptions* qosOption);
/**
* Get the profiled QoS data and executing boost value (the actual boost value during execution).
* If *profiledQoSData is nullptr, the selected Neuron Runtime would allocate *profiledQoSData.
* Otherwise, the selected Neuron Runtime would only update its fields.
* *profiledQoSData is actually allocated as a smart pointer in the selected Neuron Runtime
* instance, so the lifetime of *profiledQoSData is the same as the selected Neuron Runtime.
* Caller should be careful about the usage of *profiledQoSData,
* and never touch the allocated *profiledQoSData after NeuronDLAMuxer_release.
* @note This function is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param profiledQoSData The profiled QoS raw data.
* @param execBoostValue The executing boost value (the actual boot value set in device) based on
* scheduling policy.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getProfiledQoSData(void* dlaMuxer, ProfiledQoSData** profiledQoSData,
uint8_t* execBoostValue);
/**
* Get the physical size required by the buffer of the input tensor (specified by handle).
* Pass back the expected buffer size (byte) in *size for the tensor which holds the specified
* input handle.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getInputSize(void* dlaMuxer, uint64_t handle, size_t* size);
/**
* Get the rank required by the input tensor (specified by handle).
* Pass back the expected rank in *rank for the tensor which holds the specified input handle.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param rank The input rank.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getInputRank(void* dlaMuxer, uint64_t handle, uint32_t* rank);
/**
* Get the physical size required by the buffer of the input tensor (specified by handle) with
* hardware alignments. This function passes back the expected buffer size (byte) in *size for the
* tensor which holds the specified input handle. The value in *size has been aligned to hardware
* required size, and it can be used as ION buffer size for the specified input when
* suppressInputConversion is enabled.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getInputPaddedSize(void* dlaMuxer, uint64_t handle, size_t* size);
/**
* Get the size in pixels for each dimensions of the input tensor (specified by handle).
* This function passes back the expected size (in pixels) of each dimensions in *dim for the tensor
* which holds the specified input handle. The sizes of each dimensions in *dim have been aligned
* to hardware required sizes. When suppressInputConversion is enabled, the values in *dim are the
* required sizes of each dimensions for the specified input.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getInputPaddedDimensions(void* dlaMuxer, uint64_t handle,
RuntimeAPIDimensions* dims);
/**
* Get the physical size required by the buffer of the output tensor (specified by handle) for the
* selected DLA. This function passes back the expected buffer size (byte) in *size for the tensor
* which holds the specified output handle.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getOutputSize(void* dlaMuxer, uint64_t handle, size_t* size);
/**
* Get the physical size required by the buffer of the output tensor (specified by handle) with
* hardware alignments. This function passes back the expected buffer size (byte) in *size for the
* tensor which holds the specified output handle. The value in *size has been aligned to hardware
* required size, and it can be used as ION buffer size for the specified output when
* suppressOutputConversion is enabled.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getOutputPaddedSize(void* dlaMuxer, uint64_t handle, size_t* size);
/**
* Get the size in pixels for each dimensions of the output tensor (specified by handle).
* This function passes back the expected size (in pixels) of each dimensions in *dim for the tensor
* which holds the specified output handle. The sizes of each dimensions in *dim have been aligned
* to hardware required sizes. When suppressOutputConversion is enabled, the values in *dim are the
* required sizes of each dimensions for the specified output.
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param handle The frontend IO index.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronDLAMuxer_getOutputPaddedDimensions(void* dlaMuxer, uint64_t handle,
RuntimeAPIDimensions* dims);
/**
* Release all the runtime resources in the NeuronDLAMuxer.
* @param dlaMuxer The address of the created neuron NeuronDLAMuxer instance.
*/
void NeuronDLAMuxer_release(void* dlaMuxer);
__END_DECLS
source/backend/neuropilot/mtk/backend/api/neuron/DLAMuxerFence.h
Executable
+36
View File
@@ -0,0 +1,36 @@
#pragma once
#if __has_include("Fence.h")
#include "Fence.h"
#else
#include "neuron/api/Fence.h"
#endif
__BEGIN_DECLS
/**
* Check if the model supports fenced execution. Call this function after runtime is loaded with
* model.
* @note This function is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param supported Non-zero value indicates that the model supports fenced execution.
* @return An error code indicates whether the test model executes successfully.
*/
int NeuronDLAMuxer_isFenceSupported(void* dlaMuxer, uint8_t* supported);
/**
* Do fenced-inference. The call should return without waiting for inference to finish. The caller
* should prepare a FenceInfo structure and pass its address into this API. FenceFd in FenceInfo
* will be set, and the caller can be signaled when inference completes (or error exit) by waiting
* on the fence. Most importantly, after the fence is triggered, caller MUST call the callback in
* fenceInfo so that Neuron can perform certain post-execution tasks. The final execution status
* and inference time can be retrieved in FenceInfo after the callback is executed.
* @note This function is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
* @param dlaMuxer The address of the created NeuronDLAMuxer instance.
* @param fenceInfo The struct is used to receive the fence file descriptor and the post-inference
* callback in fenced execution.
* @return A Runtime error code.
*/
int NeuronDLAMuxer_inferenceFenced(void* dlaMuxer, FenceInfo* fenceInfo);
__END_DECLS
source/backend/neuropilot/mtk/backend/api/neuron/EnqueueTrigger.h
Executable
+64
View File
@@ -0,0 +1,64 @@
#pragma once
#if __has_include("Fence.h")
#include "Fence.h"
#else
#include "neuron/api/Fence.h"
#endif
#include <stdint.h>
__BEGIN_DECLS
// Introduction to Enqueue Trigger mechanism.
// Enqueue Trigger is an inference mechanism that can separate inference into two stage,
// (1) enqueue-stage and (2) trigger-stage.
// All the pre-execution tasks would be done in enqueue-stage. Then in trigger-stage we
// will only trigger the enqueued job to execute.
// Once the inference settings are changed. User should enqueue the job one more time.
// Then after that user can trigger that job any time they want before session released.
/**
* Check if the model supports enqueueThenTrigger execution. Call this function after
* runtime is loaded with model.
* @param runtime The address of the created neuron runtime instance.
* @param supported Non-zero value indicates that the model supports enqueueThenTrigger
* execution.
* @return An error code indicates whether the test model executes successfully.
*/
int NeuronRuntime_isEnqueueTriggerSupported(void* runtime, uint8_t* supported);
/**
* Do job-enqueue. All the pre-execution task will be done in this API.
* This API will enqueue the job then waiting for the trigger signals.
* @param runtime The address of the created neuron runtime instance.
* @return A Runtime error code.
*/
int NeuronRuntime_inferenceEnqueue(void* runtime);
/**
* Do job-trigger. Trigger job that user enqueued before. It is expected that
* NeuronRuntime_inferenceEnqueue() has been called before and it is the last API
* invoked before calling this API.
* @param runtime The address of the created neuron runtime instance.
* @return A Runtime error code.
*/
int NeuronRuntime_inferenceTrigger(void* runtime);
/**
* Do job-trigger-fenced. Trigger job that user enqueued before. It is expected that
* NeuronRuntime_inferenceEnqueue() has been called before and it is the last API
* invoked before calling this API.
* The call should return without waiting for inference to finish. The caller should
* prepare a FenceInfo structure and pass its address into this API. FenceFd in FenceInfo
* will be set and the caller can be signaled when inference completes (or error exit) by
* waiting on the fence. Most importantly, after the fence is triggered, caller MUST call
* the callback in fenceInfo so that Neuron can perform certain post-execution tasks. The
* final execution status and inference time can be retrieved in FenceInfo after the
* callback is executed.
* @param runtime The address of the created neuron runtime instance.
* @return A Runtime error code.
*/
int NeuronRuntime_inferenceTriggerFenced(void* runtime, FenceInfo* fenceInfo);
__END_DECLS
source/backend/neuropilot/mtk/backend/api/neuron/Fence.h
Executable
+63
View File
@@ -0,0 +1,63 @@
#pragma once
#include <stdint.h>
#include <sys/cdefs.h>
__BEGIN_DECLS
/**
* This struct is used to receive the fence file descriptor and the post-inference callback
* in fenced execution. Specifically, user should allocate this struct, and pass its address into
* fenced execution API. The fence FD and the call back will be set properly. After fence is
* triggered, caller can invoke the callback to retrieve execution status and execution time.
* @note This struct is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
*/
typedef struct {
/// The file descriptor of the fence to be triggered before inference.
/// Use -1 for this field if there is no inputFenceFd in the inference.
int64_t inputFenceFd;
/// The file descriptor of the fence to be triggered at the end of inference.
int64_t fenceFd;
/// Caller should call this callback after fence is triggered to retrieve execution status
/// and time. Caller should send back the address of the original FenceInfo which possesses
/// this callback in the first parameter 'opaque'.
void (*callback)(void* opaque);
/// Execution status. This will be set after callback is called.
uint32_t status;
/// Execution time. This will be set after callback is called.
uint32_t microseconds;
/// The following data are for internal use. Don't access them.
uint64_t __internal__[4];
} FenceInfo;
/**
* Check if the model supports fenced execution. Call this function after runtime is loaded with
* model.
* @note This function is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
* @param runtime The address of the created neuron runtime instance.
* @param supported Non-zero value indicates that the model supports fenced execution.
* @return An error code indicates whether the test model executes successfully.
*/
int NeuronRuntime_isFenceSupported(void* runtime, uint8_t* supported);
/**
* Do fenced-inference. The call should return without waiting for inference to finish. The caller
* should prepare a FenceInfo structure and pass its address into this API. FenceFd in FenceInfo
* will be set, and the caller can be signaled when inference completes (or error exit) by waiting
* on the fence. Most importantly, after the fence is triggered, caller MUST call the callback in
* fenceInfo so that Neuron can perform certain post-execution tasks. The final execution status
* and inference time can be retrieved in FenceInfo after the callback is executed.
* @note This function is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
* @param runtime The address of the created neuron runtime instance.
* @param fenceInfo The struct is used to receive the fence file descriptor and the post-inference
* callback in fenced execution.
* @return A Runtime error code.
*/
int NeuronRuntime_inferenceFenced(void* runtime, FenceInfo* fenceInfo);
__END_DECLS
source/backend/neuropilot/mtk/backend/api/neuron/Misc.h
Executable
+24
View File
@@ -0,0 +1,24 @@
/**
* \file
* Misc Neuron Runtime API
* ---
* Miscellaneous functionality
*/
#pragma once
#include <sys/cdefs.h>
#include "Types.h"
__BEGIN_DECLS
/**
* Get the version of Neuron runtime library.
* @note Neuron runtime can only load DLA files generated by compiler with the same major version.
* @param version the version of Neuron runtime library.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getVersion(NeuronVersion* version);
__END_DECLS
source/backend/neuropilot/mtk/backend/api/neuron/NeuronAdapter.h
Executable
+2141
View File
File diff suppressed because it is too large Load Diff
source/backend/neuropilot/mtk/backend/api/neuron/NeuronAdapterShim.h
Executable
+710
View File
@@ -0,0 +1,710 @@
/* Copyright Statement:
*
* This software/firmware and related documentation ("MediaTek Software") are
* protected under relevant copyright laws. The information contained herein
* is confidential and proprietary to MediaTek Inc. and/or its licensors.
* Without the prior written permission of MediaTek inc. and/or its licensors,
* any reproduction, modification, use or disclosure of MediaTek Software,
* and information contained herein, in whole or in part, shall be strictly prohibited.
*/
/* MediaTek Inc. (C) 2020. All rights reserved.
*
* BY OPENING THIS FILE, RECEIVER HEREBY UNEQUIVOCALLY ACKNOWLEDGES AND AGREES
* THAT THE SOFTWARE/FIRMWARE AND ITS DOCUMENTATIONS ("MEDIATEK SOFTWARE")
* RECEIVED FROM MEDIATEK AND/OR ITS REPRESENTATIVES ARE PROVIDED TO RECEIVER ON
* AN "AS-IS" BASIS ONLY. MEDIATEK EXPRESSLY DISCLAIMS ANY AND ALL WARRANTIES,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE OR NONINFRINGEMENT.
* NEITHER DOES MEDIATEK PROVIDE ANY WARRANTY WHATSOEVER WITH RESPECT TO THE
* SOFTWARE OF ANY THIRD PARTY WHICH MAY BE USED BY, INCORPORATED IN, OR
* SUPPLIED WITH THE MEDIATEK SOFTWARE, AND RECEIVER AGREES TO LOOK ONLY TO SUCH
* THIRD PARTY FOR ANY WARRANTY CLAIM RELATING THERETO. RECEIVER EXPRESSLY ACKNOWLEDGES
* THAT IT IS RECEIVER'S SOLE RESPONSIBILITY TO OBTAIN FROM ANY THIRD PARTY ALL PROPER LICENSES
* CONTAINED IN MEDIATEK SOFTWARE. MEDIATEK SHALL ALSO NOT BE RESPONSIBLE FOR ANY MEDIATEK
* SOFTWARE RELEASES MADE TO RECEIVER'S SPECIFICATION OR TO CONFORM TO A PARTICULAR
* STANDARD OR OPEN FORUM. RECEIVER'S SOLE AND EXCLUSIVE REMEDY AND MEDIATEK'S ENTIRE AND
* CUMULATIVE LIABILITY WITH RESPECT TO THE MEDIATEK SOFTWARE RELEASED HEREUNDER WILL BE,
* AT MEDIATEK'S OPTION, TO REVISE OR REPLACE THE MEDIATEK SOFTWARE AT ISSUE,
* OR REFUND ANY SOFTWARE LICENSE FEES OR SERVICE CHARGE PAID BY RECEIVER TO
* MEDIATEK FOR SUCH MEDIATEK SOFTWARE AT ISSUE.
*
* The following software/firmware and/or related documentation ("MediaTek Software")
* have been modified by MediaTek Inc. All revisions are subject to any receiver's
* applicable license agreements with MediaTek Inc.
*/
#pragma once
#include <android/log.h>
#include <dlfcn.h>
#include "NeuronAdapter.h"
#define LOAD_ADAPTER_FUNCTION(name) \
static name##_fn fn = reinterpret_cast<name##_fn>(loadAdapterFunction(#name));
#define EXECUTE_ADAPTER_FUNCTION(...) \
if (fn != nullptr) { \
fn(__VA_ARGS__); \
}
#define EXECUTE_ADAPTER_FUNCTION_RETURN_INT(...) return fn != nullptr ? fn(__VA_ARGS__) : -1;
#define EXECUTE_ADAPTER_FUNCTION_RETURN_BOOL(...) return fn != nullptr ? fn(__VA_ARGS__) : false;
#define NEURON_ADAPTER_SHIM_TAG "NeuronAdapterShim"
static void* sHandle = nullptr;
inline void* loadAdapterLibrary(const char* name) {
sHandle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
if (sHandle == nullptr) {
__android_log_print(ANDROID_LOG_ERROR, NEURON_ADAPTER_SHIM_TAG, "Unable to open library %s", name);
}
return sHandle;
}
inline void* getAdapterLibraryHandle() {
if (sHandle == nullptr) {
sHandle = loadAdapterLibrary("libneuronusdk_adapter.mtk.so");
}
if (sHandle == nullptr) {
sHandle = loadAdapterLibrary("libneuron_adapter_mgvi.so");
}
if (sHandle == nullptr) {
sHandle = loadAdapterLibrary("libneuron_adapter.so");
}
return sHandle;
}
inline void* loadAdapterFunction(const char* name) {
void* fn = nullptr;
if (getAdapterLibraryHandle() != nullptr) {
fn = dlsym(getAdapterLibraryHandle(), name);
}
if (fn == nullptr) {
__android_log_print(ANDROID_LOG_ERROR, NEURON_ADAPTER_SHIM_TAG, "Unable to open function %s", name);
}
return fn;
}
/*************************************************************************************************/
typedef int (*Neuron_getVersion_fn)(NeuronRuntimeVersion* version);
typedef int (*Neuron_getFeatureSupportedStatus_fn)(NeuronFeatureType type, bool* supported);
typedef int (*Neuron_getNeuroPilotMagicNumber_fn)(int32_t* magic);
typedef int (*Neuron_getL1MemorySizeKb_fn)(uint32_t* sizeKb);
typedef int (*NeuronModel_create_fn)(NeuronModel** model);
typedef void (*NeuronModel_free_fn)(NeuronModel* model);
typedef int (*NeuronModel_finish_fn)(NeuronModel* model);
typedef int (*NeuronModel_addOperand_fn)(NeuronModel* model, const NeuronOperandType* type);
typedef int (*NeuronModel_setOperandValue_fn)(NeuronModel* model, int32_t index, const void* buffer,
size_t length);
typedef int (*NeuronModel_setOperandValueFromModel_fn)(NeuronModel* model, int32_t index,
const NeuronModel* value);
typedef int (*NeuronModel_setOperandSymmPerChannelQuantParams_fn)(
NeuronModel* model, int32_t index, const NeuronSymmPerChannelQuantParams* channelQuant);
typedef int (*NeuronModel_setOperandPerChannelQuantParams_fn)(
NeuronModel* model, int32_t index, const NeuronPerChannelQuantParams* channelQuant);
typedef int (*NeuronModel_addOperation_fn)(NeuronModel* model, NeuronOperationType type,
uint32_t inputCount, const uint32_t* inputs,
uint32_t outputCount, const uint32_t* outputs);
typedef int (*NeuronModel_addOperationExtension_fn)(NeuronModel* model, const char* name,
const char* vendor, const NeuronDevice* device,
uint32_t inputCount, const uint32_t* inputs,
uint32_t outputCount, const uint32_t* outputs);
typedef int (*NeuronModel_identifyInputsAndOutputs_fn)(NeuronModel* model, uint32_t inputCount,
const uint32_t* inputs, uint32_t outputCount,
const uint32_t* outputs);
typedef int (*NeuronModel_getSupportedOperations_fn)(NeuronModel* model, bool* supported,
uint32_t operationCount);
typedef int (*NeuronModel_getSupportedOperationsForDevices_fn)(const NeuronModel* model,
const NeuronDevice* const* devices,
uint32_t numDevices,
bool* supportedOps);
typedef int (*NeuronModel_relaxComputationFloat32toFloat16_fn)(NeuronModel* model, bool allow);
typedef int (*NeuronModel_suppressInputConversion_fn)(NeuronModel* model, bool suppress);
typedef int (*NeuronModel_suppressOutputConversion_fn)(NeuronModel* model, bool suppress);
typedef int (*NeuronModel_restoreFromCompiledNetwork_fn)(NeuronModel** model,
NeuronCompilation** compilation,
const void* buffer, const size_t size);
typedef int (*NeuronCompilation_create_fn)(NeuronModel* model, NeuronCompilation** compilation);
typedef int (*NeuronCompilation_createForDevices_fn)(NeuronModel* model,
const NeuronDevice* const* devices,
uint32_t numDevices,
NeuronCompilation** compilation);
typedef int (*NeuronCompilation_createForDebug_fn)(NeuronModel* model,
NeuronCompilation** compilation);
typedef void (*NeuronCompilation_free_fn)(NeuronCompilation* compilation);
typedef int (*NeuronCompilation_finish_fn)(NeuronCompilation* compilation);
typedef int (*NeuronCompilation_setCaching_fn)(NeuronCompilation* compilation, const char* cacheDir,
const uint8_t* token);
typedef int (*NeuronCompilation_setPreference_fn)(NeuronCompilation* compilation,
int32_t preference);
typedef int (*NeuronCompilation_setPriority_fn)(NeuronCompilation* compilation, int32_t priority);
typedef int (*NeuronCompilation_getInputPaddedDimensions_fn)(NeuronCompilation* compilation,
int32_t index, uint32_t* dimensions);
typedef int (*NeuronCompilation_getOutputPaddedDimensions_fn)(NeuronCompilation* compilation,
int32_t index, uint32_t* dimensions);
typedef int (*NeuronCompilation_getInputPaddedSize_fn)(NeuronCompilation* compilation,
int32_t index, size_t* size);
typedef int (*NeuronCompilation_getOutputPaddedSize_fn)(NeuronCompilation* compilation,
int32_t index, size_t* size);
typedef int (*NeuronCompilation_getCompiledNetworkSize_fn)(NeuronCompilation* compilation,
size_t* size);
typedef int (*NeuronCompilation_storeCompiledNetwork_fn)(NeuronCompilation* compilation,
void* buffer, const size_t size);
typedef int (*NeuronCompilation_setOptimizationHint_fn)(NeuronCompilation* compilation,
uint32_t optimizationCode);
typedef int (*NeuronCompilation_setOptimizationString_fn)(NeuronCompilation* compilation,
const char* optimizationString);
typedef int (*NeuronCompilation_setTrimIOAlignment_fn)(NeuronCompilation* compilation, bool enable);
typedef int (*NeuronCompilation_setSWDilatedConv_fn)(NeuronCompilation* compilation, bool enable);
typedef int (*NeuronExecution_create_fn)(NeuronCompilation* compilation,
NeuronExecution** execution);
typedef void (*NeuronExecution_free_fn)(NeuronExecution* execution);
typedef int (*NeuronExecution_setInput_fn)(NeuronExecution* execution, int32_t index,
const NeuronOperandType* type, const void* buffer,
size_t length);
typedef int (*NeuronExecution_setOutput_fn)(NeuronExecution* execution, int32_t index,
const NeuronOperandType* type, void* buffer,
size_t length);
typedef int (*NeuronExecution_setInputFromMemory_fn)(NeuronExecution* execution, uint32_t index,
const NeuronOperandType* type,
const NeuronMemory* memory, size_t offset,
size_t length);
typedef int (*NeuronExecution_setOutputFromMemory_fn)(NeuronExecution* execution, uint32_t index,
const NeuronOperandType* type,
const NeuronMemory* memory, size_t offset,
size_t length);
typedef int (*NeuronMemory_createFromFd_fn)(size_t size, int protect, int fd, size_t offset,
NeuronMemory** memory);
typedef int (*NeuronMemory_createFromAHardwareBuffer_fn)(const AHardwareBuffer* ahwb,
NeuronMemory** memory);
typedef void (*NeuronMemory_free_fn)(NeuronMemory* memory);
typedef int (*NeuronExecution_compute_fn)(NeuronExecution* execution);
typedef int (*NeuronExecution_startComputeWithDependencies_fn)(
NeuronExecution* execution, const NeuronEvent* const* dependencies, uint32_t num_dependencies,
uint64_t duration, NeuronEvent** event);
typedef int (*NeuronEvent_getSyncFenceFd_fn)(const NeuronEvent* event, int* syncFenceFd);
typedef int (*NeuronEvent_wait_fn)(NeuronEvent* event);
typedef void (*NeuronEvent_free_fn)(NeuronEvent* event);
typedef int (*NeuronExecution_setLoopTimeout_fn)(NeuronExecution* execution, uint64_t duration);
typedef int (*NeuronExecution_setBoostHint_fn)(NeuronExecution* execution, uint8_t boostValue);
typedef int (*NeuronCompilation_createForMultiExecutions_fn)(NeuronModel* model,
NeuronCompilation** compilation);
typedef int (*NeuronDebug_setReportPath_fn)(NeuronModel* model, const char* path);
typedef int (*Neuron_getDeviceCount_fn)(uint32_t* numDevices);
typedef int (*Neuron_getDevice_fn)(uint32_t devIndex, NeuronDevice** device);
typedef int (*NeuronDevice_getName_fn)(const NeuronDevice* device, const char** name);
typedef int (*NeuronDevice_getDescription_fn)(const NeuronDevice* device, const char** description);
typedef int (*NeuronDevice_getExtensionSupport_fn)(const char* extensionName,
bool* isExtensionSupported);
typedef int (*NeuronModel_getExtensionOperandType_fn)(NeuronModel* model, const char* extensionName,
uint16_t operandCodeWithinExtension,
int32_t* type);
typedef int (*NeuronModel_getExtensionOperationType_fn)(NeuronModel* model,
const char* extensionName,
uint16_t operationCodeWithinExtension,
int32_t* type);
typedef int (*NeuronModel_setOperandExtensionData_fn)(NeuronModel* model, int32_t index,
const void* data, size_t length);
typedef int (*NeuronCompilation_createForBatch_fn)(NeuronModel* model,
NeuronCompilation** compilation);
typedef int (*NeuronModel_restoreFromCompiledNetworkV2_fn)(NeuronModel** model,
NeuronCompilation** compilation,
const void* buffer, const size_t size,
const CompilationType& type);
typedef int (*NeuronExecution_setRunnerPoolSize_fn)(NeuronExecution* execution, uint8_t numRunners);
typedef int (*NeuronExecution_setBatchDone_fn)(NeuronExecution* execution);
typedef int (*NeuronCompilation_createWithOptions_fn)(NeuronModel* model,
NeuronCompilation** compilation,
const char* options);
/*************************************************************************************************/
inline int Neuron_getVersion(NeuronRuntimeVersion* version) {
LOAD_ADAPTER_FUNCTION(Neuron_getVersion);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(version);
}
inline int Neuron_getFeatureSupportedStatus(NeuronFeatureType type, bool* supported) {
LOAD_ADAPTER_FUNCTION(Neuron_getFeatureSupportedStatus);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(type, supported);
}
inline int Neuron_getNeuroPilotMagicNumber(int32_t* magic) {
LOAD_ADAPTER_FUNCTION(Neuron_getNeuroPilotMagicNumber);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(magic);
}
inline int Neuron_getL1MemorySizeKb(uint32_t* sizeKb) {
LOAD_ADAPTER_FUNCTION(Neuron_getL1MemorySizeKb);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(sizeKb);
}
inline int NeuronModel_create(NeuronModel** model) {
LOAD_ADAPTER_FUNCTION(NeuronModel_create);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model);
}
inline void NeuronModel_free(NeuronModel* model) {
LOAD_ADAPTER_FUNCTION(NeuronModel_free);
EXECUTE_ADAPTER_FUNCTION(model);
}
inline int NeuronModel_finish(NeuronModel* model) {
LOAD_ADAPTER_FUNCTION(NeuronModel_finish);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model);
}
inline int NeuronModel_addOperand(NeuronModel* model, const NeuronOperandType* type) {
LOAD_ADAPTER_FUNCTION(NeuronModel_addOperand);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, type);
}
inline int NeuronModel_setOperandValue(NeuronModel* model, int32_t index, const void* buffer,
size_t length) {
LOAD_ADAPTER_FUNCTION(NeuronModel_setOperandValue);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, index, buffer, length);
}
inline int NeuronModel_setOperandValueFromModel(NeuronModel* model, int32_t index,
const NeuronModel* value) {
LOAD_ADAPTER_FUNCTION(NeuronModel_setOperandValueFromModel);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, index, value);
}
inline int NeuronModel_setOperandSymmPerChannelQuantParams(
NeuronModel* model, int32_t index, const NeuronSymmPerChannelQuantParams* channelQuant) {
LOAD_ADAPTER_FUNCTION(NeuronModel_setOperandSymmPerChannelQuantParams);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, index, channelQuant);
}
inline int NeuronModel_setOperandPerChannelQuantParams(
NeuronModel* model, int32_t index, const NeuronPerChannelQuantParams* channelQuant) {
LOAD_ADAPTER_FUNCTION(NeuronModel_setOperandPerChannelQuantParams);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, index, channelQuant);
}
inline int NeuronModel_addOperation(NeuronModel* model, NeuronOperationType type,
uint32_t inputCount, const uint32_t* inputs,
uint32_t outputCount, const uint32_t* outputs) {
LOAD_ADAPTER_FUNCTION(NeuronModel_addOperation);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, type, inputCount, inputs, outputCount, outputs);
}
inline int NeuronModel_addOperationExtension(NeuronModel* model, const char* name,
const char* vendor, const NeuronDevice* device,
uint32_t inputCount, const uint32_t* inputs,
uint32_t outputCount, const uint32_t* outputs) {
LOAD_ADAPTER_FUNCTION(NeuronModel_addOperationExtension);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, name, vendor, device, inputCount, inputs,
outputCount, outputs);
}
inline int NeuronModel_identifyInputsAndOutputs(NeuronModel* model, uint32_t inputCount,
const uint32_t* inputs, uint32_t outputCount,
const uint32_t* outputs) {
LOAD_ADAPTER_FUNCTION(NeuronModel_identifyInputsAndOutputs);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, inputCount, inputs, outputCount, outputs);
}
inline int NeuronModel_getSupportedOperations(NeuronModel* model, bool* supported,
uint32_t operationCount) {
LOAD_ADAPTER_FUNCTION(NeuronModel_getSupportedOperations);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, supported, operationCount);
}
inline int NeuronModel_getSupportedOperationsForDevices(const NeuronModel* model,
const NeuronDevice* const* devices,
uint32_t numDevices, bool* supportedOps) {
LOAD_ADAPTER_FUNCTION(NeuronModel_getSupportedOperationsForDevices);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, devices, numDevices, supportedOps);
}
inline int NeuronModel_relaxComputationFloat32toFloat16(NeuronModel* model, bool allow) {
LOAD_ADAPTER_FUNCTION(NeuronModel_relaxComputationFloat32toFloat16);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, allow);
}
inline int NeuronModel_suppressInputConversion(NeuronModel* model, bool suppress) {
LOAD_ADAPTER_FUNCTION(NeuronModel_suppressInputConversion);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, suppress);
}
inline int NeuronModel_suppressOutputConversion(NeuronModel* model, bool suppress) {
LOAD_ADAPTER_FUNCTION(NeuronModel_suppressOutputConversion);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, suppress);
}
inline int NeuronModel_restoreFromCompiledNetwork(NeuronModel** model,
NeuronCompilation** compilation,
const void* buffer, const size_t size) {
LOAD_ADAPTER_FUNCTION(NeuronModel_restoreFromCompiledNetwork);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, compilation, buffer, size);
}
// inline int NeuronModel_setName(NeuronModel* model, const char* name) {
// LOAD_ADAPTER_FUNCTION(NeuronModel_setName);
// EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, name);
// }
inline int NeuronCompilation_create(NeuronModel* model, NeuronCompilation** compilation) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_create);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, compilation);
}
// inline int NeuronCompilation_createV2(NeuronModel* model, CompilationType type, const char* options,
// NeuronCompilation** compilation) {
// LOAD_ADAPTER_FUNCTION(NeuronCompilation_createV2);
// EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, type, options, compilation);
// }
inline int NeuronCompilation_createForDevices(NeuronModel* model,
const NeuronDevice* const* devices,
uint32_t numDevices,
NeuronCompilation** compilation) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_createForDevices);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, devices, numDevices, compilation);
}
inline int NeuronCompilation_createForDebug(NeuronModel* model, NeuronCompilation** compilation) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_createForDebug);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, compilation);
}
inline void NeuronCompilation_free(NeuronCompilation* compilation) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_free);
EXECUTE_ADAPTER_FUNCTION(compilation);
}
inline int NeuronCompilation_finish(NeuronCompilation* compilation) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_finish);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation);
}
// int NeuronCompilation_getSupportedOperations(NeuronCompilation* compilation,
// uint32_t operationCount, bool* supported){
// LOAD_ADAPTER_FUNCTION(NeuronCompilation_getSupportedOperations);
// EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, operationCount, supported);
// }
inline int NeuronCompilation_setCaching(NeuronCompilation* compilation, const char* cacheDir,
const uint8_t* token) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_setCaching);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, cacheDir, token);
}
inline int NeuronCompilation_setPreference(NeuronCompilation* compilation, int32_t preference) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_setPreference);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, preference);
}
inline int NeuronCompilation_setPriority(NeuronCompilation* compilation, int32_t priority) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_setPriority);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, priority);
}
inline int NeuronCompilation_getInputPaddedDimensions(NeuronCompilation* compilation, int32_t index,
uint32_t* dimensions) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_getInputPaddedDimensions);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, index, dimensions);
}
inline int NeuronCompilation_getOutputPaddedDimensions(NeuronCompilation* compilation,
int32_t index, uint32_t* dimensions) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_getOutputPaddedDimensions);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, index, dimensions);
}
inline int NeuronCompilation_getInputPaddedSize(NeuronCompilation* compilation, int32_t index,
size_t* size) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_getInputPaddedSize);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, index, size);
}
inline int NeuronCompilation_getOutputPaddedSize(NeuronCompilation* compilation, int32_t index,
size_t* size) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_getOutputPaddedSize);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, index, size);
}
inline int NeuronCompilation_getCompiledNetworkSize(NeuronCompilation* compilation, size_t* size) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_getCompiledNetworkSize);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, size);
}
inline int NeuronCompilation_storeCompiledNetwork(NeuronCompilation* compilation, void* buffer,
const size_t size) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_storeCompiledNetwork);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, buffer, size);
}
inline int NeuronCompilation_setOptimizationHint(NeuronCompilation* compilation,
uint32_t optimizationCode) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_setOptimizationHint);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, optimizationCode);
}
inline int NeuronCompilation_setOptimizationString(NeuronCompilation* compilation,
const char* optimizationString) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_setOptimizationString);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, optimizationString);
}
inline int NeuronCompilation_setTrimIOAlignment(NeuronCompilation* compilation, bool enable) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_setTrimIOAlignment);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, enable);
}
inline int NeuronCompilation_setSWDilatedConv(NeuronCompilation* compilation, bool enable) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_setSWDilatedConv);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, enable);
}
inline int NeuronExecution_create(NeuronCompilation* compilation, NeuronExecution** execution) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_create);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(compilation, execution);
}
inline void NeuronExecution_free(NeuronExecution* execution) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_free);
EXECUTE_ADAPTER_FUNCTION(execution);
}
inline int NeuronExecution_setInput(NeuronExecution* execution, int32_t index,
const NeuronOperandType* type, const void* buffer,
size_t length) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setInput);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, index, type, buffer, length);
}
inline int NeuronExecution_setInputFromMemory(NeuronExecution* execution, uint32_t index,
const NeuronOperandType* type,
const NeuronMemory* memory, size_t offset,
size_t length) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setInputFromMemory);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, index, type, memory, offset, length);
}
inline int NeuronExecution_setOutput(NeuronExecution* execution, int32_t index,
const NeuronOperandType* type, void* buffer, size_t length) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setOutput);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, index, type, buffer, length);
}
inline int NeuronExecution_setOutputFromMemory(NeuronExecution* execution, uint32_t index,
const NeuronOperandType* type,
const NeuronMemory* memory, size_t offset,
size_t length) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setOutputFromMemory);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, index, type, memory, offset, length);
}
inline int NeuronMemory_createFromFd(size_t size, int protect, int fd, size_t offset,
NeuronMemory** memory) {
LOAD_ADAPTER_FUNCTION(NeuronMemory_createFromFd);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(size, protect, fd, offset, memory);
}
inline int NeuronMemory_createFromAHardwareBuffer(const AHardwareBuffer* ahwb,
NeuronMemory** memory) {
LOAD_ADAPTER_FUNCTION(NeuronMemory_createFromAHardwareBuffer);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(ahwb, memory);
}
inline void NeuronMemory_free(NeuronMemory* memory) {
LOAD_ADAPTER_FUNCTION(NeuronMemory_free);
EXECUTE_ADAPTER_FUNCTION(memory);
}
inline int NeuronExecution_compute(NeuronExecution* execution) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_compute);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution);
}
inline int NeuronExecution_startComputeWithDependencies(NeuronExecution* execution,
const NeuronEvent* const* dependencies,
uint32_t num_dependencies,
uint64_t duration, NeuronEvent** event) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_startComputeWithDependencies);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, dependencies, num_dependencies, duration, event);
}
inline int NeuronEvent_getSyncFenceFd(const NeuronEvent* event, int* syncFenceFd) {
LOAD_ADAPTER_FUNCTION(NeuronEvent_getSyncFenceFd);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(event, syncFenceFd);
}
inline int NeuronEvent_wait(NeuronEvent* event) {
LOAD_ADAPTER_FUNCTION(NeuronEvent_wait);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(event);
}
inline void NeuronEvent_free(NeuronEvent* event) {
LOAD_ADAPTER_FUNCTION(NeuronEvent_free);
EXECUTE_ADAPTER_FUNCTION(event);
}
inline int NeuronExecution_setLoopTimeout(NeuronExecution* execution, uint64_t duration) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setLoopTimeout);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, duration);
}
inline int NeuronExecution_setBoostHint(NeuronExecution* execution, uint8_t boostValue) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setBoostHint);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, boostValue);
}
inline int NeuronCompilation_createForMultiExecutions(NeuronModel* model,
NeuronCompilation** compilation) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_createForMultiExecutions);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, compilation);
}
inline int NeuronDebug_setReportPath(NeuronModel* model, const char* path) {
LOAD_ADAPTER_FUNCTION(NeuronDebug_setReportPath);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, path);
}
inline int Neuron_getDeviceCount(uint32_t* numDevices) {
LOAD_ADAPTER_FUNCTION(Neuron_getDeviceCount);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(numDevices);
}
inline int Neuron_getDevice(uint32_t devIndex, NeuronDevice** device) {
LOAD_ADAPTER_FUNCTION(Neuron_getDevice);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(devIndex, device);
}
inline int NeuronDevice_getName(const NeuronDevice* device, const char** name) {
LOAD_ADAPTER_FUNCTION(NeuronDevice_getName);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(device, name);
}
inline int NeuronDevice_getDescription(const NeuronDevice* device, const char** description) {
LOAD_ADAPTER_FUNCTION(NeuronDevice_getDescription);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(device, description);
}
inline int NeuronDevice_getExtensionSupport(const char* extensionName, bool* isExtensionSupported) {
LOAD_ADAPTER_FUNCTION(NeuronDevice_getExtensionSupport);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(extensionName, isExtensionSupported);
}
inline int NeuronModel_getExtensionOperandType(NeuronModel* model, const char* extensionName,
uint16_t operandCodeWithinExtension, int32_t* type) {
LOAD_ADAPTER_FUNCTION(NeuronModel_getExtensionOperandType);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, extensionName, operandCodeWithinExtension, type);
}
inline int NeuronModel_getExtensionOperationType(NeuronModel* model, const char* extensionName,
uint16_t operationCodeWithinExtension,
int32_t* type) {
LOAD_ADAPTER_FUNCTION(NeuronModel_getExtensionOperationType);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, extensionName, operationCodeWithinExtension, type);
}
inline int NeuronModel_setOperandExtensionData(NeuronModel* model, int32_t index, const void* data,
size_t length) {
LOAD_ADAPTER_FUNCTION(NeuronModel_setOperandExtensionData);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, index, data, length);
}
inline int NeuronCompilation_createForBatch(NeuronModel* model, NeuronCompilation** compilation) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_createForBatch);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, compilation);
}
inline int NeuronModel_restoreFromCompiledNetworkV2(NeuronModel** model,
NeuronCompilation** compilation,
const void* buffer, const size_t size,
const CompilationType& type) {
LOAD_ADAPTER_FUNCTION(NeuronModel_restoreFromCompiledNetworkV2);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, compilation, buffer, size, type);
}
inline int NeuronExecution_setRunnerPoolSize(NeuronExecution* execution, uint8_t numRunners) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setRunnerPoolSize);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution, numRunners);
}
inline int NeuronExecution_setBatchDone(NeuronExecution* execution) {
LOAD_ADAPTER_FUNCTION(NeuronExecution_setBatchDone);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(execution);
}
inline int NeuronCompilation_createWithOptions(NeuronModel* model, NeuronCompilation** compilation,
const char* options) {
LOAD_ADAPTER_FUNCTION(NeuronCompilation_createWithOptions);
EXECUTE_ADAPTER_FUNCTION_RETURN_INT(model, compilation, options);
}
source/backend/neuropilot/mtk/backend/api/neuron/RuntimeAPI.h
Executable
+475
View File
@@ -0,0 +1,475 @@
/**
* \file
* Neuron Runtime API
* ---
* Neuron provides some APIs to create runtime environment, parse compiled model file,
* and do inference with a network.
* \n The Runtime user should include this header to use Runtime API.
* Note that some APIs that set input and output info need the user to specify the handle of
* the input/output tensor that he/she wants to set.\n The user may
* \n 1) Acts as ANN or TFLite, which always know the handle
* \n 2) Run a precompiled network. The user should understand the model in the beginning.
* \n 3) Run a precompiled network without knowing what the network look like. In this case,
* it is impossible for the user to do inference without taking a glance at the network
* IO map info. \n Otherwise, the user cannot even give a valid input with valid input shape.
* After the user checks the IO map, they would also acquire the handle and the corresponding
* shape.
*/
#pragma once
#if __has_include("Types.h")
#include "Types.h"
#else
#include "neuron/api/Types.h"
#endif
#include <stddef.h>
#include <stdint.h>
#include <sys/cdefs.h>
__BEGIN_DECLS
/// This option controls if the underlying hardware should split and run a graph across homogeneous
/// devices. Note that this does not control the heterogeneous parallelism in the Runtime software.
/// \warning This option is to be deprecated in Neuron 6.0
typedef enum {
Auto = 0, ///< Scheduler decide
Single = 1, ///< Force single MDLA
Dual = 2, ///< Force multi MDLA
} MDLACoreMode;
typedef struct {
/// Device kind can be chosen from kEnvOptNullDevice, or kEnvOptHardware.
/// \n For hardware development, use kEnvOptHardware.
uint32_t deviceKind;
/// Set MDLA core option.
/// \warning This option is no longer effective. To be removed in Neuron 6.0
MDLACoreMode MDLACoreOption;
/// Hint CPU backends to use \#threads for execution
uint8_t CPUThreadNum;
/// Set this to true to bypass preprocess and feed data in the format that the device demands.
bool suppressInputConversion;
/// Set this to true to bypass postprocess and retrieve raw device output.
bool suppressOutputConversion;
} EnvOptions;
/// For unsigned char deviceKind.
const unsigned char kEnvOptNullDevice = 1 << 0;
const unsigned char kEnvOptHardware = 1 << 2;
const unsigned char kEnvOptPredictor = 1 << 3;
/**
* @param options The environment options for the Neuron Runtime.
* @return 1 to indicate user-specified EnvOptions use a NullDevice. Otherwise, return 0.
*/
inline int IsNullDevice(const EnvOptions* options) {
return options->deviceKind & kEnvOptNullDevice;
}
/**
* @param options The environment options for the Neuron Runtime.
* @return 1 to indicate user-specified EnvOptions use real hardware. Otherwise, return 0.
*/
inline int IsHardware(const EnvOptions* options) {
return options->deviceKind & kEnvOptHardware;
}
/**
* Create a Neuron Runtime based on the setting specified in options. The address of the created
* instance will be passed back in *runtime.
* @param optionsToDeprecate The environment options for the Neuron Runtime (To be deprecated).
* @param runtime Runtime provides API for applications to run a compiled network on specified
* input.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_create(const EnvOptions* optionsToDeprecate, void** runtime);
/**
* Create a Neuron Runtime based on the setting specified in options. The address of the created
* instance will be passed back in *runtime.
* @param options The environment options for the Neuron Runtime.
* @param optionsToDeprecate The environment options for the Neuron Runtime (To be deprecated).
* @param runtime Runtime provides API for applications to run a compiled network on specified
* input.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_create_with_options(const char* options,
const EnvOptions* optionsToDeprecate,
void** runtime);
/**
* Clone an existing, loaded Neuron Runtime. The existing Runtime must be DLA-loaded, and the DLA
* file must exist, too. The address of the created instance (copy) will be passed by in
* *newRuntime. Note that the constant data buffer may be shared with the clone runtime,
* reducing memory footprint.
* @param oldRuntime The existing, loaded Neuron Runtime.
* @param newRuntime The Neuron Runtime clone.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_clone(void* oldRuntime, void** newRuntime);
/**
* Load the compiled network from dla file.
* @param runtime The address of the created neuron runtime instance.
* @param pathToDlaFile The dla file path.
* @return A RuntimeAPI error code. 0 indicating load network successfully.
*/
int NeuronRuntime_loadNetworkFromFile(void* runtime, const char* pathToDlaFile);
/**
* Load the compiled network from a memory buffer.
* @param runtime The address of the created neuron runtime instance.
* @param buffer The memory buffer.
* @param size The size of the buffer.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_loadNetworkFromBuffer(void* runtime, const void* buffer, size_t size);
/**
* Set the memory buffer for the tensor which hold the specified input handle in the original
* network. If there are multiple inputs, each of them have to be set.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param buffer The input buffer.
* @param length The input buffer size.
* @param attribute The buffer attribute for setting ION.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setInput(void* runtime, uint64_t handle, const void* buffer, size_t length,
BufferAttribute attribute);
/**
* Set the memory buffer and offset for the tensor which hold the specified input handle in the
* original network. If there are multiple inputs, each of them have to be set.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param buffer The input buffer.
* @param length The input buffer size. This length doesn't include offset.
* @param attribute The buffer attribute for setting ION.
* @param offset The offset for ION buffer.
* @param offset Reading ION buffer from start addr + offset.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setOffsetedInput(void* runtime, uint64_t handle, const void* buffer,
size_t length, BufferAttribute attribute, size_t offset);
/**
* If there is only one input, this function can set the buffer to the input automatically.
* Otherwise, NEURONRUNTIME_INCOMPLETE is returned.
* @param runtime The address of the created neuron runtime instance.
* @param buffer The input buffer.
* @param length The input buffer size.
* @param attribute The buffer attribute for setting ION.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setSingleInput(void* runtime, const void* buffer, size_t length,
BufferAttribute attribute);
/**
* Set shape for the input tensor which hold the specified input handle in the
* original network. If there are multiple inputs with dynamic shapes, each of
* them have to be set. This API is only used when input is dynamic shape, otherwise
* error code will be returned.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param dims A array of dimension sizes for each dimension. For NHWC, dims[0] is N.
* @param rank The input rank. For exmaple, rank is 4 for NHWC.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setInputShape(void* runtime, uint64_t handle, uint32_t* dims, uint32_t rank);
/**
* Set the memory buffer for the tensor which hold the specified output handle in the original
* network. If there are multiple outputs, each of them have to be set.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param buffer The output buffer.
* @param length The output buffer size.
* @param attribute The buffer attribute for setting ION.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setOutput(void* runtime, uint64_t handle, void* buffer, size_t length,
BufferAttribute attribute);
/**
* Set the memory buffer and offset for the tensor which hold the specified output handle in the
* original network. If there are multiple outputs, each of them have to be set.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param buffer The output buffer.
* @param length The output buffer size. This length doesn't include offset.
* @param attribute The buffer attribute for setting ION.
* @param offset Writing ION buffer from start addr + offset.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setOffsetedOutput(void* runtime, uint64_t handle, void* buffer, size_t length,
BufferAttribute attribute, size_t offset);
/**
* If there is only one output, this function can set the buffer to the output automatically.
* Otherwise, NEURONRUNTIME_INCOMPLETE is returned.
* @param runtime The address of the created neuron runtime instance.
* @param buffer The output buffer.
* @param length The output buffer size.
* @param attribute The buffer attribute for setting ION.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setSingleOutput(void* runtime, void* buffer, size_t length,
BufferAttribute attribute);
/**
* Set the QoS configuration for Neuron Runtime.
* If qosOption.profiledQoSData is not nullptr,
* Neuron Runtime would use it as the profiled QoS data.
* @param runtime The address of the created neuron runtime instance.
* @param qosOption The option for QoS configuration.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_setQoSOption(void* runtime, const QoSOptions* qosOption);
/**
* Get the number of inputs of the model in the runtime. The number of inputs will be passed
* back in *size
* @param runtime The address of the created NeuronRuntime instance.
* @param size The pointer to a size_t to store the passed back value.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getInputNumber(void* runtime, size_t* size);
/**
* Get the physical size required by the buffer of the input tensor (specified by handle).
* Pass back the expected buffer size (byte) in *size for the tensor which holds the specified
* input handle.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getInputSize(void* runtime, uint64_t handle, size_t* size);
/**
* Get the rank required by the input tensor (specified by handle).
* Pass back the expected rank in *rank for the tensor which holds the specified input handle.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param rank The input rank.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getInputRank(void* runtime, uint64_t handle, uint32_t* rank);
/**
* If there is only one input, this function can get the physical size required by the buffer of
* input and return the expected buffer size (byte) in *size.
* Otherwise, NEURONRUNTIME_INCOMPLETE is returned.
* @param runtime The address of the created neuron runtime instance.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getSingleInputSize(void* runtime, size_t* size);
/**
* Get the physical size required by the buffer of the input tensor (specified by handle) with
* hardware alignments. This function passes back the expected buffer size (byte) in *size for the
* tensor which holds the specified input handle. The value in *size has been aligned to hardware
* required size, and it can be used as ION buffer size for the specified input when
* suppressInputConversion is enabled.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getInputPaddedSize(void* runtime, uint64_t handle, size_t* size);
/**
* If there is only one input, this function passes back the expected size (byte) of its buffer in
* *size. The value in *size has been aligned to hardware required size, and it can be used as ION
* buffer size for input when suppressInputConversion is enabled. Otherwise, the returned value is
* NEURONRUNTIME_INCOMPLETE.
* @param runtime The address of the created neuron runtime instance.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getSingleInputPaddedSize(void* runtime, size_t* size);
/**
* Get the size in pixels for each dimensions of the input tensor (specified by handle).
* This function passes back the expected size (in pixels) of each dimensions in *dim for the tensor
* which holds the specified input handle. The sizes of each dimensions in *dim have been aligned
* to hardware required sizes. When suppressInputConversion is enabled, the values in *dim are the
* required sizes of each dimensions for the specified input.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getInputPaddedDimensions(void* runtime, uint64_t handle,
RuntimeAPIDimensions* dims);
/**
* Get the size in pixels for each dimensions of the only input. This function passes back the
* expected size (in pixels) of each dimensions in *dim. The sizes of each dimensions in *dim have
* been aligned to hardware required sizes.
* If suppressInputConversion is enabled, the values in *dim are the required sizes of each
* dimensions for input. Otherwise NEURONRUNTIME_INCOMPLETE is returned.
* @param runtime The address of the created neuron runtime instance.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getSingleInputPaddedDimensions(void* runtime, RuntimeAPIDimensions* dims);
/**
* Get the number of outputs of the model in the runtime. The number of outputs will be
* passed back in *size
* @param runtime The address of the created NeuronRuntime instance.
* @param size The pointer to a size_t to store the passed back value.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getOutputNumber(void* runtime, size_t* size);
/**
* Get the physical size required by the buffer of the output tensor (specified by handle).
* This function passes back the expected buffer size (byte) in *size for the tensor which holds the
* specified output handle.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getOutputSize(void* runtime, uint64_t handle, size_t* size);
/**
* Get the physical size required by the buffer of the only output.
* If there is only one Output, this function passes back the expected size (byte) of its buffer
* in *size. Otherwise, NEURONRUNTIME_INCOMPLETE is returned.
* @param runtime The address of the created neuron runtime instance.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getSingleOutputSize(void* runtime, size_t* size);
/**
* Get the physical size required by the buffer of the output tensor (specified by handle) with
* hardware alignments. This function passes back the expected buffer size (byte) in *size for the
* tensor which holds the specified output handle. The value in *size has been aligned to hardware
* required size, and it can be used as ION buffer size for the specified output when
* suppressOutputConversion is enabled.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getOutputPaddedSize(void* runtime, uint64_t handle, size_t* size);
/**
* Get the physical size required by the buffer of the only output with hardware alignments.
* If there is only one Output, this function passes back the expected size (byte) of its buffer in
* *size. The value in *size has been aligned to hardware required size, and it can be used as ION
* buffer size for output when suppressOutputConversion is enabled. Otherwise, the returned value is
* NEURONRUNTIME_INCOMPLETE.
* @param runtime The address of the created neuron runtime instance.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getSingleOutputPaddedSize(void* runtime, size_t* size);
/**
* Get the size in pixels for each dimensions of the output tensor (specified by handle).
* This function passes back the expected size (in pixels) of each dimensions in *dim for the tensor
* which holds the specified output handle. The sizes of each dimensions in *dim have been aligned
* to hardware required sizes. When suppressOutputConversion is enabled, the values in *dim are the
* required sizes of each dimensions for the specified output.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getOutputPaddedDimensions(void* runtime, uint64_t handle,
RuntimeAPIDimensions* dims);
/**
* Get the size in pixels for each dimensions of the only output. If there is only one Output, this
* function passes back the expected size (in pixels) of each dimensions in *dim. The sizes of each
* dimensions in *dim have been aligned to hardware required sizes. If suppressOutputConversion is
* enabled, the values in *dim are the required sizes of each dimensions for output. Otherwise,
* NEURONRUNTIME_INCOMPLETE is returned.
* @param runtime The address of the created neuron runtime instance.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getSingleOutputPaddedDimensions(void* runtime, RuntimeAPIDimensions* dims);
/**
* Get the rank required by the output tensor (specified by handle).
* Pass back the expected rank in *rank for the tensor which holds the specified output handle.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param rank The output rank.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getOutputRank(void* runtime, uint64_t handle, uint32_t* rank);
/**
* Get the profiled QoS data and executing boost value (the actual boost value during execution).
* If *profiledQoSData is nullptr, Neuron Runtime would allocate *profiledQoSData.
* Otherwise, Neuron Runtime would only update its fields.
* *profiledQoSData is actually allocated as a smart pointer in Neuron Runtime instance,
* so the lifetime of *profiledQoSData is the same as Neuron Runtime.
* Caller should be careful about the usage of *profiledQoSData,
* and never touch the allocated *profiledQoSData after NeuronRuntime_release.
* @note This function is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
* @param runtime The address of the created neuron runtime instance.
* @param profiledQoSData The profiled QoS raw data.
* @param execBoostValue The executing boost value (the actual boot value set in device) based on
* scheduling policy.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getProfiledQoSData(void* runtime, ProfiledQoSData** profiledQoSData,
uint8_t* execBoostValue);
/**
* Do inference.
* @param runtime The address of the created neuron runtime instance.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_inference(void* runtime);
/**
* Release the runtime resource.
* @param runtime The address of the created neuron runtime instance.
*/
void NeuronRuntime_release(void* runtime);
/**
* Get the version of Neuron runtime library.
* @note Neuron runtime can only load DLA files generated by compiler with the same major version.
* @param version the version of Neuron runtime library.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getVersion(NeuronVersion* version);
/**
* Get metadata info in dla file, which is provided through compiler option --dla-metadata.
* @param runtime The address of the created neuron runtime instance.
* @param key The key for the target data
* @param size The size of the target data. If there is no corresponding metadata, size is 0.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getMetadataInfo(void* runtime, const char* key, size_t* size);
/**
* Get metadata in dla file, which is provided through compiler option --dla-metadata.
* @param runtime The address of the created neuron runtime instance.
* @param key The key for the target data
* @param data The destination data buffer.
* @param size The size to read from metadata.
* @return A RuntimeAPI error code.
*/
int NeuronRuntime_getMetadata(void* runtime, const char* key, char* data, size_t size);
__END_DECLS
source/backend/neuropilot/mtk/backend/api/neuron/RuntimeV2.h
Executable
+359
View File
@@ -0,0 +1,359 @@
/**
* \file
* RuntimeV2.
* ---
* NeuronRuntimeV2 API allows user to create a NeuronRuntimeV2 from the specified .DLA file.
* Users can enqueue asynchronous inference requests into the created runtime. Or, users can issue
* conventional synchronous requests, too.
*/
#pragma once
#include "Types.h"
#include <stddef.h>
#include <stdint.h>
#include <sys/cdefs.h>
__BEGIN_DECLS
/// IOBuffer is a descriptor describing the buffer which will be used as an inference
/// input or output. Users should zero the whole IOBuffer, then fill those fields with valid
/// data.
typedef struct IOBuffer {
void* buffer;
size_t length;
int fd; // Normal buffer must have -1 for this. Otherwise, fill with imported buffer FD
int offset; // Offset for imported buffer FD
uint64_t reserved2_should_be_init_zero;
uint64_t reserved3_should_be_init_zero;
#ifdef __cplusplus
IOBuffer(void* buffer, size_t length, int fd, int offset = 0)
: buffer(buffer), length(length), fd(fd), offset(offset),
reserved2_should_be_init_zero(0),
reserved3_should_be_init_zero(0) {}
#endif
} IOBufferType;
/// AsyncInferenceRequest represents a single inference request to be enqueued into Runtime
/// Note that all the data pointed by pointers in AsyncInferenceRequest must remain valid
/// until the inference of that request is complete.
typedef struct {
/// A pointer to the array of input buffer descriptions. The number of elements should
/// equal to the result of NeuronRuntimeV2_getInputNumber();
IOBuffer* inputs;
/// A pointer to the array of output buffer descriptions. The number of elements should
/// equal to the result of NeuronRuntimeV2_getOutputNumber();
IOBuffer* outputs;
/// A callback function specified by the user for the runtime to notify inference complete.
/// When it's called, the ID of the job just have finished and the opaque pointer in the
/// original request will be passed back in 'job_id' and 'opaque'. The execution status
/// is given by 'status'. A zero status indicates success. Otherwise, the inference job
/// has failed.
void (*finish_cb)(uint64_t job_id, void* opaque, int status);
/// A pointer to an opaque data, which will be passed back when finish_cb is called.
void* opaque;
} AsyncInferenceRequest;
/// SyncInferenceRequest represents a synchronous inference request to run in the Runtime.
/// The call will block until the inference finishes.
typedef struct {
/// A pointer to the array of input buffer descriptions. The number of elements should
/// equal to the result of NeuronRuntimeV2_getInputNumber();
IOBuffer* inputs;
/// A pointer to the array of output buffer descriptions. The number of elements should
/// equal to the result of NeuronRuntimeV2_getOutputNumber();
IOBuffer* outputs;
} SyncInferenceRequest;
/**
* Create a NeuronRuntimeV2 based on the setting specified in options. It acts as a thread
* pool, waiting to accept AsyncInferenceRequest or SyncInferenceRequest on a DLA file.
* When the runtime receives a request, it enqueues the request into its backlog ring buffer,
* and the internal load balancer will dispatch the request to the appropriate thread for
* execution. However, there is no guarantee on the order of completion of
* AsyncInferenceRequest. The user-specified callback should be aware of this.
* SyncInferenceRequest, on the other hand, always block until the request finishes.
* The address of the created runtime instance will be passed back in *runtime.
* @param pathToDlaFile The DLA file path.
* @param nbThreads The number of threads in the runtime. Large value for 'nbThread' could result
in a large memory footprint. 'nbThread' is the number of working threads and
each thread would maintain its own working buffer, so the total memory footprint
of all threads could be large.
* @param runtime The pointer will be modified to the created NeuronRuntimeV2 instance on success.
* @param backlog The maximum size of the backlog ring buffer. In most cases, using 2048 is enough.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_create(const char* pathToDlaFile, size_t nbThreads, void** runtime,
size_t backlog);
/**
* Like NeuronRuntimeV2_create(), but it takes an additional option string.
* @param pathToDlaFile The DLA file path.
* @param nbThreads The number of threads in the runtime. Large value for 'nbThread' could result
in a large memory footprint. 'nbThread' is the number of working threads and
each thread would maintain its own working buffer, so the total memory footprint
of all threads could be large.
* @param runtime The pointer will be modified to the created NeuronRuntimeV2 instance on success.
* @param backlog The maximum size of the backlog ring buffer. In most cases, using 2048 is enough.
* @param options A null-terminated C-string specifying runtime options.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_create_with_options(const char* pathToDlaFile, size_t nbThreads, void** runtime,
size_t backlog, const char* options);
/**
* Like NeuronRuntimeV2_create(), but it creates the Runtime instance from a memory buffer
* containing the DLA data.
* @param buffer The DLA data buffer.
* @param len The DLA data buffer size.
* @param nbThreads The number of threads in the runtime. Large value for 'nbThread' could result
in a large memory footprint. 'nbThread' is the number of working threads and
each thread would maintain its own working buffer, so the total memory footprint
of all threads could be large.
* @param runtime The pointer will be modified to the created NeuronRuntimeV2 instance on success.
* @param backlog The maximum size of the backlog ring buffer. In most cases, using 2048 is enough.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_createFromBuffer(const void* buffer, size_t len, size_t nbThreads,
void** runtime, size_t backlog);
/**
* Like NeuronRuntimeV2_createFromBuffer(), but it takes an additional option string.
* containing the DLA data.
* @param buffer The DLA data buffer.
* @param len The DLA data buffer size.
* @param nbThreads The number of threads in the runtime. Large value for 'nbThread' could result
in a large memory footprint. 'nbThread' is the number of working threads and
each thread would maintain its own working buffer, so the total memory footprint
of all threads could be large.
* @param runtime The pointer will be modified to the created NeuronRuntimeV2 instance on success.
* @param backlog The maximum size of the backlog ring buffer. In most cases, using 2048 is enough.
* @param options A null-terminated C-string specifying runtime options.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_createFromBuffer_with_options(const void* buffer, size_t len, size_t nbThreads,
void** runtime, size_t backlog,
const char* options);
/**
* Release the runtime. Calling this function will block until all requests finish.
* @param runtime The address of the created NeuronRuntimeV2 instance.
*/
void NeuronRuntimeV2_release(void* runtime);
/**
* Enqueue one AsyncInferenceRequest. If the backlog ring buffer is not full, this
* function returns immediately, and the runtime will execute the request asynchronously. If
* the backlog is full (due to back pressure from execution), this call will block until the
* backlog ring buffer releases at least one available slot for the request. A unique ID is
* returned for the enqueued request in *job_id. The ID sequence starts from zero and
* increases with each received request. The 2^64 capacity for job ID should be enough for
* any applications.
* @param runtime The address of the created NeuronRuntimeV2 instance.
* @param request The asynchronous inference request
* @param job_id The ID for this request is filled into *job_id. Later the ID will be passed
* back when the finish_cb is called.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_enqueue(void* runtime, AsyncInferenceRequest request, uint64_t* job_id);
/**
* Perform a synchronous inference request. The request will be also enqueued into the
* Runtime ring buffer as NeuronRuntimeV2_enqueue() does. However, the call will block until
* the request finishes.
* @param runtime The address of the created NeuronRuntimeV2 instance.
* @param request The synchronous inference request
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_run(void* runtime, SyncInferenceRequest request);
/**
* Get the number of inputs of the model in the runtime. The number of inputs will be passed
* back in *size
* @param runtime The address of the created NeuronRuntimeV2 instance.
* @param size The pointer to a size_t to store the passed back value.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getInputNumber(void* runtime, size_t* size);
/**
* Get the number of outputs of the model in the runtime. The number of outputs will be
* passed back in *size
* @param runtime The address of the created NeuronRuntimeV2 instance.
* @param size The pointer to a size_t to store the passed back value.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getOutputNumber(void* runtime, size_t* size);
/**
* Get the rank required by the input tensor (specified by handle).
* Pass back the expected rank in *rank for the tensor which holds the specified input handle.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param rank The input rank.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getInputRank(void* runtime, uint64_t handle, uint32_t* rank);
/**
* Get the physical size required by the buffer of the input tensor (specified by handle).
* Pass back the expected buffer size (byte) in *size for the tensor which holds the specified
* input handle.
* @param runtime The address of the created NeuronRuntimeV2 instance.
* @param handle The frontend IO index.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getInputSize(void* runtime, uint64_t handle, size_t* size);
/**
* Get the physical size required by the buffer of the output tensor (specified by handle).
* This funxtion passes back the expected buffer size (byte) in *size for the tensor which holds the
* specified output handle.
* @param runtime The address of the created NeuronRuntimeV2 instance.
* @param handle The frontend IO index.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getOutputSize(void* runtime, uint64_t handle, size_t* size);
/**
* Get the physical size required by the buffer of the input tensor (specified by handle) with
* hardware alignments. This function passes back the expected buffer size (byte) in *size for the
* tensor which holds the specified input handle. The value in *size has been aligned to hardware
* required size, and it can be used as ION buffer size for the specified input when
* suppressInputConversion is enabled.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param size The input buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getInputPaddedSize(void* runtime, uint64_t handle, size_t* size);
/**
* Get the size in pixels for each dimensions of the input tensor (specified by handle).
* This function passes back the expected size (in pixels) of each dimensions in *dim for the tensor
* which holds the specified input handle. The sizes of each dimensions in *dim have been aligned
* to hardware required sizes. When suppressInputConversion is enabled, the values in *dim are the
* required sizes of each dimensions for the specified input.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getInputPaddedDimensions(void* runtime, uint64_t handle,
RuntimeAPIDimensions* dims);
/**
* Get the physical size required by the buffer of the output tensor (specified by handle) with
* hardware alignments. This function passes back the expected buffer size (byte) in *size for the
* tensor which holds the specified output handle. The value in *size has been aligned to hardware
* required size, and it can be used as ION buffer size for the specified output when
* suppressOutputConversion is enabled.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param size The output buffer size.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getOutputPaddedSize(void* runtime, uint64_t handle, size_t* size);
/**
* Get the size in pixels for each dimensions of the output tensor (specified by handle).
* This function passes back the expected size (in pixels) of each dimensions in *dim for the tensor
* which holds the specified output handle. The sizes of each dimensions in *dim have been aligned
* to hardware required sizes. When suppressOutputConversion is enabled, the values in *dim are the
* required sizes of each dimensions for the specified output.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param dims The size (in pixels) of each dimensions.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getOutputPaddedDimensions(void* runtime, uint64_t handle,
RuntimeAPIDimensions* dims);
/**
* Get the rank required by the output tensor (specified by handle).
* Pass back the expected rank in *rank for the tensor which holds the specified output handle.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param rank The output rank.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getOutputRank(void* runtime, uint64_t handle, uint32_t* rank);
/**
* Set shape for the input tensor which hold the specified input handle in the
* original network. If there are multiple inputs with dynamic shapes, each of
* them have to be set. This API is only used when input is dynamic shape, otherwise
* error code will be returned.
* @param runtime The address of the created neuron runtime instance.
* @param handle The frontend IO index.
* @param dims A array of dimension sizes for each dimension. For NHWC, dims[0] is N.
* @param rank The input rank. For exmaple, rank is 4 for NHWC.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_setInputShape(void* runtime, uint64_t handle, uint32_t* dims, uint32_t rank);
/**
* Set the QoS configuration for Neuron Runtime. If qosOption.profiledQoSData is not null,
* Neuron Runtime would use it to store the profiled QoS data.
* *** Note : qosOption.profiledQoSData has no effect at all.
* *** Note : Using this API when NeuronRuntimeV2 is working leads to undefined behavior.
* Namely, this API should be used only when all requests have finished and no
* new request is being issued.
* @param runtime The address of the created neuron runtime instance.
* @param qosOption The option for QoS configuration.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_setQoSOption(void* runtime, const QoSOptions* qosOption);
/**
* Get the profiled QoS data and executing boost value (the actual boost value during execution).
* If *profiledQoSData is nullptr, Neuron Runtime would allocate *profiledQoSData.
* Otherwise, Neuron Runtime would only update its fields.
* *profiledQoSData is actually allocated as a smart pointer in Neuron Runtime instance,
* so the lifetime of *profiledQoSData is the same as Neuron Runtime.
* Caller should be careful about the usage of *profiledQoSData,
* and never touch the allocated *profiledQoSData after NeuronRuntime_release.
*
* *** Note : Only effective when NeuronRuntimeV2 has nbThreads = 1.
* *** Note : Using this API when NeuronRuntimeV2 is working leads to undefined behavior.
* Namely, this API should be used only when all requests have finished and no
* new request is being issued.
* @note This function is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
* @param runtime The address of the created neuron runtime instance.
* @param profiledQoSData The profiled QoS raw data.
* @param execBoostValue The executing boost value (the actual boot value set in device) based on
* scheduling policy.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getProfiledQoSData(void* runtime, ProfiledQoSData** profiledQoSData,
uint8_t* execBoostValue);
/**
* Get metadata info in dla file, which is provided through compiler option --dla-metadata.
* @param runtime The address of the created neuron runtime instance.
* @param key The key for the target data
* @param size The size of the target data. If there is no corresponding metadata, size is 0.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getMetadataInfo(void* runtime, const char* key, size_t* size);
/**
* Get metadata in dla file, which is provided through compiler option --dla-metadata.
* @param runtime The address of the created neuron runtime instance.
* @param key The key for the target data
* @param data The destination data buffer.
* @param size The size to read from metadata.
* @return A RuntimeAPI error code.
*/
int NeuronRuntimeV2_getMetadata(void* runtime, const char* key, char* data, size_t size);
__END_DECLS
source/backend/neuropilot/mtk/backend/api/neuron/Types.h
Executable
+202
View File
@@ -0,0 +1,202 @@
/**
* \file
* Types.h
* ---
* Common type definitions.
*/
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <sys/cdefs.h>
__BEGIN_DECLS
/// Execution preference.
/// @note This enum is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
typedef enum {
/// Prefer performance.
NEURONRUNTIME_PREFER_PERFORMANCE = 0,
/// Prefer low power.
NEURONRUNTIME_PREFER_POWER,
/// Hint for turbo boost mode.
/// Only valid for certain platforms (e.g., DX-1),
/// For other platforms without turbo boost mode support, the behavior of
/// NEURONRUNTIME_HINT_TURBO_BOOST would be identical to NEURONRUNTIME_PREFER_PERFORMANCE.
NEURONRUNTIME_HINT_TURBO_BOOST,
} RuntimeAPIQoSPreference;
/// Task priority.
typedef enum {
NEURONRUNTIME_PRIORITY_LOW = 0, ///< Low priority.
NEURONRUNTIME_PRIORITY_MED, ///< Medium priority.
NEURONRUNTIME_PRIORITY_HIGH, ///< High priority.
} RuntimeAPIQoSPriority;
/// Special boost value hint.
/// @note This enum is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
typedef enum {
/// 101: Hint to notify the scheduler to use the profiled boost value.
NEURONRUNTIME_BOOSTVALUE_PROFILED = 101,
NEURONRUNTIME_BOOSTVALUE_MAX = 100, ///< 100: Maximum boost value
NEURONRUNTIME_BOOSTVALUE_MIN = 0, ///< 0: Minimum boost value
} RuntimeAPIQoSBoostValue;
/// Delayed power off time.
/// @note This enum is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
typedef enum {
/// Default power off time
NEURONRUNTIME_POWER_OFF_TIME_DEFAULT = -1,
} RuntimeAPIQoSDelayedPowerOffTime;
/// Power policy.
/// @note This enum is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
typedef enum {
/// Default policy.
NEURONRUNTIME_POWER_POLICY_DEFAULT = 0,
NEURONRUNTIME_POWER_POLICY_SUSTAINABLE = 1,
NEURONRUNTIME_POWER_POLICY_PERFORMANCE = 2,
NEURONRUNTIME_POWER_POLICY_POWER_SAVING = 3,
} RuntimeAPIQoSPowerPolicy;
/// Application type.
/// @note This enum is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
typedef enum {
/// Normal type.
NEURONRUNTIME_APP_NORMAL = 0,
} RuntimeAPIQoSAppType;
/**
* Raw data for QoS configuration.
* All of those fields should be filled with the profiled data.
* @note This struct is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
*/
typedef struct {
/// <b> Profiled execution time </b> :
/// the profiled execution time (in usec).
uint64_t execTime;
/// <b> Suggested time </b>:
/// the suggested time (in msec).
uint32_t suggestedTime;
/// <b> Profled bandwidh </b>:
/// the profiled bandwidh (in MB/s).
uint32_t bandwidth;
/// <b> Profiled boost value </b>:
/// the profiled executing boost value (range in 0 to 100).
uint8_t boostValue;
} QoSData;
/// Maintain the profiled QoS raw data.
typedef struct {
/// Maintain profiled QoS raw data in a pointer of pointer.\n
/// This field could be nullptr if there is no previous profiled data.
QoSData** qosData;
/// Number of sub-command in *qosData.\n
/// This field could be nullptr if there is no previous profiled data.
uint32_t* numSubCmd;
/// Number of subgraph.\n
/// This field should be zero if there is no previous profiled data.
uint32_t numSubgraph;
} ProfiledQoSData;
/// QoS Option for configuration.
typedef struct {
/// <b>Execution preference</b>:\n
/// NEURONRUNTIME_PREFER_PERFORMANCE, NEURONRUNTIME_PREFER_POWER,
/// or NEURONRUNTIME_HINT_TURBO_BOOST.
RuntimeAPIQoSPreference preference;
/// <b>Task priority</b>: \n NEURONRUNTIME_PRIORITY_HIGH, NEURONRUNTIME_PRIORITY_MED,
/// or NEURONRUNTIME_PRIORITY_LOW.
RuntimeAPIQoSPriority priority;
/// <b>Boost value hint</b>: hint for the device frequency, ranged between 0 (lowest) to 100
/// (highest). This value is the hint for baseline boost value in the scheduler,
/// which sets the executing boost value (the actual boot value set in device) based on
/// scheduling policy.
/// For the inferences with preference set as NEURONRUNTIME_PREFER_PERFORMANCE, scheduler
/// guarantees that the executing boost value would not be lower than the boost value hint.
/// On the other hand, for the inferences with preference set as NEURONRUNTIME_PREFER_POWER,
/// scheduler would try to save power by configuring the executing boost value with some value
/// that is not higher than the boost value hint.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
uint8_t boostValue;
/// <b>Maximum boost value</b>: reserved.
/// Assign 0 to this field by default.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
uint8_t maxBoostValue;
/// <b>Minimum boost value</b>: reserved.
/// Assign 0 to this field by default.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
uint8_t minBoostValue;
/// \b Deadline: deadline for the inference (in msec).
/// Setting any non-zero value would nofity the scheduler that this inference is
/// a real-time task.
/// This field should be zero, unless this inference is a real-time task.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
uint16_t deadline;
/// <b>Abort time</b>: the maximum inference time for the inference (in msec).
/// If the inference is not completed before the abort time, the scheduler would
/// abort the inference.
/// This field should be zero, unless you wish to abort the inference.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
uint16_t abortTime;
/// <b>Delayed power off time</b>: delayed power off time after inference completed (in msec).
/// Scheduler would start a timer for the time interval defined in delayed power off time
/// after the inference completion. Once the delayed power off time expired and there is no
/// other incoming inference requests, the underlying devices would be powered off for
/// power-saving purpose.
/// Set this field to NEURONRUNTIME_POWER_OFF_TIME_DEFAULT to use the default power off policy
/// in the scheduler.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
int32_t delayedPowerOffTime;
/// <b>Power policy</b>: configure power policy for scheduler.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
RuntimeAPIQoSPowerPolicy powerPolicy;
/// <b>Application type</b>: hint for the application type for the inference.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
RuntimeAPIQoSAppType applicationType;
/// <b>Profiled QoS Data</b>: pointer to the historical QoS data of previous inferences.
/// If there is no profiled data, this field could be nullptr.
/// For the details, please check the ProfiledQoSData part.
/// @note This member is not supported on MediaTek TV platforms (MT99XX/MT96XX/MT76XX/MT58XX).
ProfiledQoSData* profiledQoSData;
} QoSOptions;
// The dimension size of RuntimeAPIDimensions.
const uint32_t kDimensionSize = 4;
/// The aligned sizes of dimensions. For NHWC format, dimensions[0] is N.
typedef struct {
uint32_t dimensions[kDimensionSize];
} RuntimeAPIDimensions;
/// The structure to represent the neuron version.
typedef struct {
uint8_t major;
uint8_t minor;
uint8_t patch;
} NeuronVersion;
/// A Neuron Runtime API returns an error code to show the status of execution.
typedef enum {
NEURONRUNTIME_NO_ERROR = 0, ///< 0: The API is complete successfully.
NEURONRUNTIME_OUT_OF_MEMORY = 1, ///< 1: Memory is not enough for the API.
NEURONRUNTIME_INCOMPLETE = 2, ///< 2: Not in use.
NEURONRUNTIME_UNEXPECTED_NULL = 3, ///< 3: A required pointer is null.
NEURONRUNTIME_BAD_DATA = 4, ///< 4: Failed to load data or set input/output.
NEURONRUNTIME_BAD_STATE = 5, ///< 5: Not in use.
NEURONRUNTIME_RUNTIME_ERROR = 6, ///< 6: Hardware or simulator return unexpectedly.
} RuntimeAPIErrorCode;
/// BufferAttribute is used to inform the runtime whether this buffer is an ION buffer. If ionFd is
/// -1, the buffer is a non-ION buffer. Otherwise, the buffer is an ION buffer and ionFd is its
/// shared ION buffer file descriptor. Android device implementations may benefit from this
/// information to eliminate unnecessary data copy.
typedef struct {
int ionFd; ///< -1: Non-ION buffer.
} BufferAttribute;
#define NON_ION_FD -1
__END_DECLS
source/backend/neuropilot/mtk/backend/backend.cpp
Executable
+111
View File
@@ -0,0 +1,111 @@
#include "backend/backend.h"
#include "backend/neuron_api.h"
#include "common/logging.h"
#include <dlfcn.h>
#include <sys/mman.h>
#include <unistd.h>
#define LOG_TAG_LOAD "neuron_runtime_library_load_func"
#define LOG_TAG_INIT "neuron_runtime_library_init"
namespace mtk { namespace backend {
inline void* load_func(void* handle, const char* func_name) {
// Load the function specified by func_name, and exit if the loading is failed.
void* func_ptr = dlsym(handle, func_name);
if (func_name == nullptr) {
LOG(ERROR, LOG_TAG_LOAD) << "Fail to find function: " << func_name;
} else {
LOG(DEBUG, LOG_TAG_LOAD) << "Found function: " << func_name;
}
return func_ptr;
}
namespace neuron_api {
FnNeuronRuntime_create fnNeuronRuntime_create;
FnNeuronRuntime_create_with_options fnNeuronRuntime_create_with_options;
FnNeuronRuntime_loadNetworkFromFile fnNeuronRuntime_loadNetworkFromFile;
FnNeuronRuntime_loadNetworkFromBuffer fnNeuronRuntime_loadNetworkFromBuffer;
FnNeuronRuntime_setInput fnNeuronRuntime_setInput;
FnNeuronRuntime_setOutput fnNeuronRuntime_setOutput;
FnNeuronRuntime_setOffsetedInput fnNeuronRuntime_setOffsetedInput;
FnNeuronRuntime_setOffsetedOutput fnNeuronRuntime_setOffsetedOutput;
FnNeuronRuntime_setQoSOption fnNeuronRuntime_setQoSOption;
FnNeuronRuntime_getInputSize fnNeuronRuntime_getInputSize;
FnNeuronRuntime_getOutputSize fnNeuronRuntime_getOutputSize;
FnNeuronRuntime_getInputPaddedSize fnNeuronRuntime_getInputPaddedSize;
FnNeuronRuntime_getOutputPaddedSize fnNeuronRuntime_getOutputPaddedSize;
FnNeuronRuntime_getInputPaddedDimensions fnNeuronRuntime_getInputPaddedDimensions;
FnNeuronRuntime_getOutputPaddedDimensions fnNeuronRuntime_getOutputPaddedDimensions;
FnNeuronRuntime_getInputNumber fnNeuronRuntime_getInputNumber;
FnNeuronRuntime_getOutputNumber fnNeuronRuntime_getOutputNumber;
FnNeuronRuntime_getProfiledQoSData fnNeuronRuntime_getProfiledQoSData;
FnNeuronRuntime_inference fnNeuronRuntime_inference;
FnNeuronRuntime_release fnNeuronRuntime_release;
FnNeuronRuntime_getVersion fnNeuronRuntime_getVersion;
FnNeuronRuntime_inferenceEnqueue fnNeuronRuntime_inferenceEnqueue;
FnNeuronRuntime_inferenceTrigger fnNeuronRuntime_inferenceTrigger;
bool load_library() {
static bool neuron_runtime_loaded = false;
if (neuron_runtime_loaded) {
LOG(DEBUG, LOG_TAG_INIT) << "Skip loading neuron runtime again.";
return true;
}
LOG(DEBUG, LOG_TAG_INIT) << "Begin loading neuron runtime.";
bool status = true;
// Load neuron runtime
LOG(DEBUG, LOG_TAG_INIT) << "dlopen neuron_runtime";
void* rt_handle = dlopen("libneuron_runtime.so", RTLD_LAZY);
if (rt_handle == nullptr) {
LOG(ERROR, LOG_TAG_INIT) << "Failed to load neuron";
status = false;
} else {
LOG(DEBUG, LOG_TAG_INIT) << "Load neuron OK";
}
#define LOAD(name) \
fn##name = reinterpret_cast<Fn##name>(load_func(rt_handle, #name)); \
if (fn##name == nullptr) { \
status = false; \
}
LOAD(NeuronRuntime_create)
LOAD(NeuronRuntime_create_with_options)
LOAD(NeuronRuntime_loadNetworkFromFile)
LOAD(NeuronRuntime_loadNetworkFromBuffer)
LOAD(NeuronRuntime_setInput)
LOAD(NeuronRuntime_setOutput)
LOAD(NeuronRuntime_setOffsetedInput)
LOAD(NeuronRuntime_setOffsetedOutput)
LOAD(NeuronRuntime_setQoSOption)
LOAD(NeuronRuntime_getInputSize)
LOAD(NeuronRuntime_getOutputSize)
LOAD(NeuronRuntime_getInputPaddedSize)
LOAD(NeuronRuntime_getOutputPaddedSize)
LOAD(NeuronRuntime_getInputPaddedDimensions)
LOAD(NeuronRuntime_getOutputPaddedDimensions)
LOAD(NeuronRuntime_getInputNumber)
LOAD(NeuronRuntime_getOutputNumber)
LOAD(NeuronRuntime_getProfiledQoSData)
LOAD(NeuronRuntime_inference)
LOAD(NeuronRuntime_release)
LOAD(NeuronRuntime_getVersion)
LOAD(NeuronRuntime_inferenceEnqueue)
LOAD(NeuronRuntime_inferenceTrigger)
#undef LOAD
neuron_runtime_loaded = true;
return status;
}
} // namespace neuron_api
} } // namespace mtk::backend

Some files were not shown because too many files have changed in this diff Show More