/* * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "priorBoxPlugin.h" #include #include #include #include #include #include #include using namespace nvinfer1; using nvinfer1::plugin::PriorBox; using nvinfer1::plugin::PriorBoxPluginCreator; namespace { const char* PRIOR_BOX_PLUGIN_VERSION{"1"}; const char* PRIOR_BOX_PLUGIN_NAME{"PriorBox_TRT"}; } // namespace PluginFieldCollection PriorBoxPluginCreator::mFC{}; std::vector PriorBoxPluginCreator::mPluginAttributes; // Constructor PriorBox::PriorBox(PriorBoxParameters param, int32_t H, int32_t W) : mParam(param) , mH(H) , mW(W) { // each obj should manage its copy of param auto copyParamData = [](float*& dest, const float* src, const size_t size) { if (size > 0) { dest = new float[size]; std::copy_n(src, size, dest); } else { ASSERT(dest == nullptr); } }; copyParamData(mParam.minSize, param.minSize, param.numMinSize); copyParamData(mParam.maxSize, param.maxSize, param.numMaxSize); copyParamData(mParam.aspectRatios, param.aspectRatios, param.numAspectRatios); setupDeviceMemory(); } void PriorBox::setupDeviceMemory() noexcept { auto copyToDevice = [](const void* hostData, size_t count) -> Weights { void* deviceData = nullptr; CUASSERT(cudaMalloc(&deviceData, count * sizeof(float))); CUASSERT(cudaMemcpy(deviceData, hostData, count * sizeof(float), cudaMemcpyHostToDevice)); return Weights{DataType::kFLOAT, deviceData, int64_t(count)}; }; // minSize is required and needs to be non-negative ASSERT(mParam.numMinSize > 0 && mParam.minSize != nullptr); for (auto i = 0; i < mParam.numMinSize; ++i) { ASSERT(mParam.minSize[i] > 0 && "minSize must be positive"); } minSize = copyToDevice(mParam.minSize, mParam.numMinSize); ASSERT(mParam.numAspectRatios >= 0 && mParam.aspectRatios != nullptr); // Aspect ratio of 1.0 is built in. std::vector tmpAR(1, 1); for (auto i = 0; i < mParam.numAspectRatios; ++i) { float ar = mParam.aspectRatios[i]; bool alreadyExist = false; // Prevent duplicated aspect ratios from input for (unsigned j = 0; j < tmpAR.size(); ++j) { if (std::fabs(ar - tmpAR[j]) < 1e-6) { alreadyExist = true; break; } } if (!alreadyExist) { tmpAR.push_back(ar); if (mParam.flip) { tmpAR.push_back(1.0F / ar); } } } /* * aspectRatios is of type nvinfer1::Weights * https://docs.nvidia.com/deeplearning/sdk/tensorrt-api/c_api/classnvinfer1_1_1_weights.html * aspectRatios.count is different to mParam.numAspectRatios */ aspectRatios = copyToDevice(&tmpAR[0], tmpAR.size()); // Number of prior boxes per grid cell on the feature map // tmpAR already included an aspect ratio of 1.0 mNumPriors = tmpAR.size() * mParam.numMinSize; /* * If we have maxSizes, as long as all the maxSizes meets assertion requirement, we add one bounding box per maxSize * The final number of prior boxes per grid cell on feature map * mNumPriors = * tmpAR.size() * mParam.numMinSize If numMaxSize == 0 * (tmpAR.size() + 1) * mParam.numMinSize If mParam.numMinSize == mParam.numMaxSize */ if (mParam.numMaxSize > 0) { ASSERT(mParam.numMinSize == mParam.numMaxSize && mParam.maxSize != nullptr); for (auto i = 0; i < mParam.numMaxSize; ++i) { // maxSize should be greater than minSize ASSERT(mParam.maxSize[i] > mParam.minSize[i] && "maxSize must be greater than minSize"); mNumPriors++; } maxSize = copyToDevice(mParam.maxSize, mParam.numMaxSize); } } PriorBox::PriorBox(const void* data, size_t length) { const char *d = static_cast(data), *a = d; mParam = read(d); auto readArray = [&d](const int32_t size, float*& array) { if (size > 0) { array = new float[size]; for (auto i = 0; i < size; i++) { array[i] = read(d); } } else { array = nullptr; } }; readArray(mParam.numMinSize, mParam.minSize); readArray(mParam.numMaxSize, mParam.maxSize); readArray(mParam.numAspectRatios, mParam.aspectRatios); mH = read(d); mW = read(d); ASSERT(d == a + length); setupDeviceMemory(); } // Returns the number of output from the plugin layer int32_t PriorBox::getNbOutputs() const noexcept { // Number of outputs from the plugin layer is 1 return 1; } // Computes and returns the output dimensions Dims PriorBox::getOutputDimensions(int32_t index, const Dims* inputs, int32_t nbInputDims) noexcept { ASSERT(nbInputDims == 2); // Only one output from the plugin layer ASSERT(index == 0); // Particularity of the PriorBox layer: no batchSize dimension needed mH = inputs[0].d[1], mW = inputs[0].d[2]; // workaround for TRT // The first channel is for prior box coordinates. // The second channel is for prior box scaling factors, which is simply a copy of the variance provided. return Dims3(2, mH * mW * mNumPriors * 4, 1); } int32_t PriorBox::initialize() noexcept { return STATUS_SUCCESS; } size_t PriorBox::getWorkspaceSize(int32_t /*maxBatchSize*/) const noexcept { return 0; } int32_t PriorBox::enqueue(int32_t /*batchSize*/, const void* const* /*inputs*/, void* const* outputs, void* /*workspace*/, cudaStream_t stream) noexcept { void* outputData = outputs[0]; pluginStatus_t status = priorBoxInference(stream, mParam, mH, mW, mNumPriors, aspectRatios.count, minSize.values, maxSize.values, aspectRatios.values, outputData); return status; } // Returns the size of serialized parameters size_t PriorBox::getSerializationSize() const noexcept { // PriorBoxParameters, minSize, maxSize, aspectRatios, mH, mW - the construct parameters return sizeof(PriorBoxParameters) + sizeof(float) * (mParam.numMinSize + mParam.numMaxSize + mParam.numAspectRatios) + sizeof(int) * 2; } void PriorBox::serialize(void* buffer) const noexcept { char *d = static_cast(buffer), *a = d; write(d, mParam); auto writeArray = [&d](const int32_t size, const float* array) { for (auto i = 0; i < size; i++) { write(d, array[i]); } }; writeArray(mParam.numMinSize, mParam.minSize); writeArray(mParam.numMaxSize, mParam.maxSize); writeArray(mParam.numAspectRatios, mParam.aspectRatios); write(d, mH); write(d, mW); ASSERT(d == a + getSerializationSize()); } bool PriorBox::supportsFormat(DataType type, PluginFormat format) const noexcept { return (type == DataType::kFLOAT && format == PluginFormat::kLINEAR); } const char* PriorBox::getPluginType() const noexcept { return PRIOR_BOX_PLUGIN_NAME; } const char* PriorBox::getPluginVersion() const noexcept { return PRIOR_BOX_PLUGIN_VERSION; } void PriorBox::destroy() noexcept { CUASSERT(cudaFree(const_cast(minSize.values))); if (mParam.numMaxSize > 0) { CUASSERT(cudaFree(const_cast(maxSize.values))); } if (mParam.numAspectRatios > 0) { CUASSERT(cudaFree(const_cast(aspectRatios.values))); } delete[] mParam.minSize; delete[] mParam.maxSize; delete[] mParam.aspectRatios; delete this; } IPluginV2Ext* PriorBox::clone() const noexcept { PriorBox* obj = new PriorBox(mParam, mH, mW); obj->setPluginNamespace(mPluginNamespace.c_str()); return obj; } // Set plugin namespace void PriorBox::setPluginNamespace(const char* pluginNamespace) noexcept { mPluginNamespace = pluginNamespace; } const char* PriorBox::getPluginNamespace() const noexcept { return mPluginNamespace.c_str(); } // Return the DataType of the plugin output at the requested index. DataType PriorBox::getOutputDataType(int32_t index, const nvinfer1::DataType* /*inputTypes*/, int32_t /*nbInputs*/) const noexcept { // Two outputs ASSERT(index == 0 || index == 1); return DataType::kFLOAT; } // Return true if output tensor is broadcast across a batch. bool PriorBox::isOutputBroadcastAcrossBatch(int32_t /*outputIndex*/, const bool* /*inputIsBroadcasted*/, int32_t /*nbInputs*/) const noexcept { return false; } // Return true if plugin can use input that is broadcast across batch without replication. bool PriorBox::canBroadcastInputAcrossBatch(int32_t /*inputIndex*/) const noexcept { return false; } // Configure the layer with input and output data types. void PriorBox::configurePlugin(const Dims* inputDims, int32_t nbInputs, const Dims* outputDims, int32_t nbOutputs, const DataType* inputTypes, const DataType* /*outputTypes*/, const bool* /*inputIsBroadcast*/, const bool* /*outputIsBroadcast*/, PluginFormat floatFormat, int32_t /*maxBatchSize*/) noexcept { ASSERT(*inputTypes == DataType::kFLOAT && floatFormat == PluginFormat::kLINEAR); ASSERT(nbInputs == 2); ASSERT(nbOutputs == 1); ASSERT(inputDims[0].nbDims == 3); ASSERT(inputDims[1].nbDims == 3); ASSERT(outputDims[0].nbDims == 3); mH = inputDims[0].d[1]; mW = inputDims[0].d[2]; // prepare for the inference function if (mParam.imgH == 0 || mParam.imgW == 0) { mParam.imgH = inputDims[1].d[1]; mParam.imgW = inputDims[1].d[2]; } if (mParam.stepH == 0 || mParam.stepW == 0) { mParam.stepH = static_cast(mParam.imgH) / mH; mParam.stepW = static_cast(mParam.imgW) / mW; } } // Attach the plugin object to an execution context and grant the plugin the access to some context resource. void PriorBox::attachToContext(cudnnContext* /*cudnnContext*/, cublasContext* /*cublasContext*/, IGpuAllocator* /*gpuAllocator*/) noexcept {} // Detach the plugin object from its execution context. void PriorBox::detachFromContext() noexcept {} PriorBoxPluginCreator::PriorBoxPluginCreator() { mPluginAttributes.clear(); mPluginAttributes.emplace_back(PluginField("minSize", nullptr, PluginFieldType::kFLOAT32, 1)); mPluginAttributes.emplace_back(PluginField("maxSize", nullptr, PluginFieldType::kFLOAT32, 1)); mPluginAttributes.emplace_back(PluginField("aspectRatios", nullptr, PluginFieldType::kFLOAT32, 1)); mPluginAttributes.emplace_back(PluginField("flip", nullptr, PluginFieldType::kINT32, 1)); mPluginAttributes.emplace_back(PluginField("clip", nullptr, PluginFieldType::kINT32, 1)); mPluginAttributes.emplace_back(PluginField("variance", nullptr, PluginFieldType::kFLOAT32, 4)); mPluginAttributes.emplace_back(PluginField("imgH", nullptr, PluginFieldType::kINT32, 1)); mPluginAttributes.emplace_back(PluginField("imgW", nullptr, PluginFieldType::kINT32, 1)); mPluginAttributes.emplace_back(PluginField("stepH", nullptr, PluginFieldType::kFLOAT32, 1)); mPluginAttributes.emplace_back(PluginField("stepW", nullptr, PluginFieldType::kFLOAT32, 1)); mPluginAttributes.emplace_back(PluginField("offset", nullptr, PluginFieldType::kFLOAT32, 1)); mFC.nbFields = mPluginAttributes.size(); mFC.fields = mPluginAttributes.data(); } PriorBoxPluginCreator::~PriorBoxPluginCreator() { // Free allocated memory (if any) here } const char* PriorBoxPluginCreator::getPluginName() const noexcept { return PRIOR_BOX_PLUGIN_NAME; } const char* PriorBoxPluginCreator::getPluginVersion() const noexcept { return PRIOR_BOX_PLUGIN_VERSION; } const PluginFieldCollection* PriorBoxPluginCreator::getFieldNames() noexcept { return &mFC; } IPluginV2Ext* PriorBoxPluginCreator::createPlugin(const char* /*name*/, const PluginFieldCollection* fc) noexcept { const PluginField* fields = fc->fields; PriorBoxParameters params; std::unique_ptr minSize; std::unique_ptr maxSize; std::unique_ptr aspectRatios; for (auto i = 0; i < fc->nbFields; ++i) { const char* attrName = fields[i].name; if (!strcmp(attrName, "minSize")) { ASSERT(fields[i].type == PluginFieldType::kFLOAT32); const int32_t size = fields[i].length; params.numMinSize = size; if (size > 0) { minSize.reset(new float[size]); params.minSize = minSize.get(); const auto* minS = static_cast(fields[i].data); for (auto j = 0; j < size; j++) { params.minSize[j] = *minS; minS++; } } else { params.minSize = nullptr; } } else if (!strcmp(attrName, "maxSize")) { ASSERT(fields[i].type == PluginFieldType::kFLOAT32); const int32_t size = fields[i].length; params.numMaxSize = size; if (size > 0) { maxSize.reset(new float[size]); params.maxSize = maxSize.get(); const auto* maxS = static_cast(fields[i].data); for (auto j = 0; j < size; j++) { params.maxSize[j] = *maxS; maxS++; } } else { params.maxSize = nullptr; } } else if (!strcmp(attrName, "aspectRatios")) { ASSERT(fields[i].type == PluginFieldType::kFLOAT32); const int32_t size = fields[i].length; params.numAspectRatios = size; if (size > 0) { aspectRatios.reset(new float[size]); params.aspectRatios = aspectRatios.get(); const auto* aR = static_cast(fields[i].data); for (auto j = 0; j < size; j++) { params.aspectRatios[j] = *aR; aR++; } } else { params.aspectRatios = nullptr; } } else if (!strcmp(attrName, "variance")) { ASSERT(fields[i].type == PluginFieldType::kFLOAT32); const int32_t size = fields[i].length; const auto* lVar = static_cast(fields[i].data); for (auto j = 0; j < size; j++) { params.variance[j] = (*lVar); lVar++; } } else if (!strcmp(attrName, "flip")) { ASSERT(fields[i].type == PluginFieldType::kINT32); params.flip = static_cast(*(static_cast(fields[i].data))); } else if (!strcmp(attrName, "clip")) { ASSERT(fields[i].type == PluginFieldType::kINT32); params.clip = static_cast(*(static_cast(fields[i].data))); } else if (!strcmp(attrName, "imgH")) { ASSERT(fields[i].type == PluginFieldType::kINT32); params.imgH = static_cast(*(static_cast(fields[i].data))); } else if (!strcmp(attrName, "imgW")) { ASSERT(fields[i].type == PluginFieldType::kINT32); params.imgW = static_cast(*(static_cast(fields[i].data))); } else if (!strcmp(attrName, "stepH")) { ASSERT(fields[i].type == PluginFieldType::kFLOAT32); params.stepH = static_cast(*(static_cast(fields[i].data))); } else if (!strcmp(attrName, "stepW")) { ASSERT(fields[i].type == PluginFieldType::kFLOAT32); params.stepW = static_cast(*(static_cast(fields[i].data))); } else if (!strcmp(attrName, "offset")) { ASSERT(fields[i].type == PluginFieldType::kFLOAT32); params.offset = static_cast(*(static_cast(fields[i].data))); } } PriorBox* obj = new PriorBox(params); obj->setPluginNamespace(mNamespace.c_str()); return obj; } IPluginV2Ext* PriorBoxPluginCreator::deserializePlugin( const char* /*name*/, const void* serialData, size_t serialLength) noexcept { // This object will be deleted when the network is destroyed, which will // call PriorBox::destroy() PriorBox* obj = new PriorBox(serialData, serialLength); obj->setPluginNamespace(mNamespace.c_str()); return obj; }