github-featured/nvidia--tensorrt
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

10.8 release changes

Browse Source 
Signed-off-by: Yuan Yao <yuanyao@nvidia.com>
This commit is contained in:
Yuan Yao
2025-01-31 10:17:00 -08:00
parent 97ff24489d
commit a180e08111
265 changed files with 1147239 additions and 1166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/docstrings/infer/pyAlgorithmSelectorDoc.h
+11 -1
View File
@@ -23,6 +23,7 @@ namespace tensorrt
namespace IAlgorithmIOInfoDOC
{
constexpr const char* descr = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
This class carries information about input or output of the algorithm.
IAlgorithmIOInfo for all the input and output along with IAlgorithmVariant denotes the variation of algorithm
and can be used to select or reproduce an algorithm using IAlgorithmSelector.select_algorithms().
@@ -37,6 +38,7 @@ constexpr const char* descr = R"trtdoc(
namespace IAlgorithmVariantDOC
{
constexpr const char* descr = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
provides a unique 128-bit identifier, which along with the input and output information
denotes the variation of algorithm and can be used to select or reproduce an algorithm,
using IAlgorithmSelector.select_algorithms()
@@ -52,6 +54,7 @@ constexpr const char* descr = R"trtdoc(
namespace IAlgorithmContextDoc
{
constexpr const char* descr = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
Describes the context and requirements, that could be fulfilled by one or
more instances of IAlgorithm.
see IAlgorithm
@@ -60,9 +63,11 @@ constexpr const char* descr = R"trtdoc(
:ivar num_inputs: :class:`int` number of inputs of the algorithm.
:ivar num_outputs: :class:`int` number of outputs of the algorithm.
)trtdoc"
;
;
constexpr const char* get_shape = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
Get the minimum / optimum / maximum dimensions for a dynamic input tensor.
:arg index: Index of the input or output of the algorithm. Incremental numbers assigned to indices of inputs and the outputs.
@@ -74,6 +79,7 @@ constexpr const char* get_shape = R"trtdoc(
namespace IAlgorithmDoc
{
constexpr const char* descr = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
Application-implemented interface for selecting and reporting the tactic selection of a layer.
Tactic Selection is a step performed by the builder for deciding best algorithms for a layer.
@@ -83,6 +89,7 @@ constexpr const char* descr = R"trtdoc(
)trtdoc";
constexpr const char* get_algorithm_io_info = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
A single call for both inputs and outputs. Incremental numbers assigned to indices of inputs and the outputs.
:arg index: Index of the input or output of the algorithm. Incremental numbers assigned to indices of inputs and the outputs.
@@ -94,6 +101,7 @@ constexpr const char* get_algorithm_io_info = R"trtdoc(
namespace IAlgorithmSelectorDoc
{
constexpr const char* descr = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
Interface implemented by application for selecting and reporting algorithms of a layer provided by the
builder.
note A layer in context of algorithm selection may be different from ILayer in INetworkDefiniton.
@@ -109,6 +117,7 @@ constexpr const char* descr = R"trtdoc(
)trtdoc";
constexpr const char* select_algorithms = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
Select Algorithms for a layer from the given list of algorithm choices.
Note: TRT uses its default algorithm selection to choose from the list returned by the user.
@@ -130,6 +139,7 @@ constexpr const char* select_algorithms = R"trtdoc(
)trtdoc";
constexpr const char* report_algorithms = R"trtdoc(
[DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead.
Called by TensorRT to report choices it made.
Note: For a given optimization profile, this call comes after all calls to select_algorithms.
python/docstrings/infer/pyCoreDoc.h
+90 -3
View File
@@ -1026,6 +1026,7 @@ constexpr char const* GPU_FALLBACK
constexpr char const* REFIT = R"trtdoc(Enable building a refittable engine)trtdoc";
constexpr char const* DISABLE_TIMING_CACHE
= R"trtdoc(Disable reuse of timing information across identical layers.)trtdoc";
constexpr char const* EDITABLE_TIMING_CACHE = R"trtdoc(Enable the editable timing cache.)trtdoc";
constexpr char const* TF32
= R"trtdoc(Allow (but not require) computations on tensors of type DataType.FLOAT to use TF32. TF32 computes inner products by rounding the inputs to 10-bit mantissas before multiplying, but accumulates the sum using 23-bit mantissas. Enabled by default.)trtdoc";
constexpr char const* SPARSE_WEIGHTS
@@ -1040,7 +1041,7 @@ constexpr char const* DIRECT_IO
= R"trtdoc(Require that no reformats be inserted between a layer and a network I/O tensor for which ``ITensor.allowed_formats`` was set. Build fails if a reformat is required for functional correctness.
[DEPRECATED] Deprecated in TensorRT 10.7.))trtdoc";
constexpr char const* REJECT_EMPTY_ALGORITHMS
= R"trtdoc(Fail if IAlgorithmSelector.select_algorithms returns an empty set of algorithms.)trtdoc";
= R"trtdoc([DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead. Fail if IAlgorithmSelector.select_algorithms returns an empty set of algorithms.)trtdoc";
constexpr char const* VERSION_COMPATIBLE
= R"trtdoc(Restrict to lean runtime operators to provide version forward compatibility for the plan files.)trtdoc";
constexpr char const* EXCLUDE_LEAN_RUNTIME = R"trtdoc(Exclude lean runtime from the plan.)trtdoc";
@@ -1063,6 +1064,7 @@ constexpr char const* WEIGHT_STREAMING
constexpr char const* INT4 = R"trtdoc(Enable plugins with INT4 input/output)trtdoc";
constexpr char const* STRICT_NANS
= R"trtdoc(Disable floating-point optimizations: 0*x => 0, x-x => 0, or x/x => 1. These identities are not true when x is a NaN or Inf, and thus might hide propagation or generation of NaNs.)trtdoc";
constexpr char const* FP4 = R"trtdoc(Enable plugins with FP4 input/output)trtdoc";
constexpr char const* MONITOR_MEMORY = R"trtdoc(Enable memory monitor during build time.)trtdoc";
} // namespace BuilderFlagDoc
@@ -1072,7 +1074,7 @@ constexpr char const* descr = R"trtdoc(The type for memory pools used by TensorR
constexpr char const* WORKSPACE = R"trtdoc(
WORKSPACE is used by TensorRT to store intermediate buffers within an operation.
This defaults to max device memory. Set to a smaller value to restrict tactics that use over the threshold en masse.
For more targeted removal of tactics use the IAlgorithmSelector interface.
For more targeted removal of tactics use the IAlgorithmSelector interface ([DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead).
)trtdoc";
constexpr char const* DLA_MANAGED_SRAM = R"trtdoc(
DLA_MANAGED_SRAM is a fast software managed RAM used by DLA to communicate within a layer.
@@ -1170,6 +1172,25 @@ constexpr char const* WINDOWS_AMD64 = R"trtdoc(
)trtdoc";
} // namespace RuntimePlatformDoc
namespace TilingOptimizationLevelDoc
{
constexpr char const* descr = R"trtdoc(
Describes the optimization level of tiling strategies. A higher level allows TensorRT to spend more time searching for better tiling strategy.
)trtdoc";
constexpr char const* NONE = R"trtdoc(
Do not apply any tiling strategy.
)trtdoc";
constexpr char const* FAST = R"trtdoc(
Use a fast algorithm and heuristic based strategy. Slightly increases engine build time.
)trtdoc";
constexpr char const* MODERATE = R"trtdoc(
Increase search space and use a mixed heuristic/profiling strategy. Moderately increases engine build time.
)trtdoc";
constexpr char const* FULL = R"trtdoc(
Increase search space even wider. Significantly increases engine build time.
)trtdoc";
} // namespace TilingOptimizationLevelDoc
namespace NetworkDefinitionCreationFlagDoc
{
constexpr char const* descr
@@ -1300,6 +1321,34 @@ constexpr char const* ONELINE = R"trtdoc(Print layer information in one line per
constexpr char const* JSON = R"trtdoc(Print layer information in JSON format.)trtdoc";
} // namespace LayerInformationFormatDoc
namespace TimingCacheKeyDoc
{
constexpr char const* descr = R"trtdoc(
The key to retrieve timing cache entries.
)trtdoc";
constexpr char const* parse = R"trtdoc(
Parse the text into a `TimingCacheKey` object.
:arg text: The input text.
:returns: A `TimingCacheKey` object.
)trtdoc";
constexpr char const* convertTimingCacheKeyToString = R"trtdoc(
Convert a `TimingCacheKey` object into text.
:returns: A `str` object.
)trtdoc";
} // namespace TimingCacheKeyDoc
namespace TimingCacheValueDoc
{
constexpr char const* descr = R"trtdoc(
The values in the cache entry.
)trtdoc";
}
namespace ITimingCacheDoc
{
constexpr char const* descr = R"trtdoc(
@@ -1331,6 +1380,42 @@ constexpr char const* reset = R"trtdoc(
:returns: A `bool` indicating whether the reset operation is done successfully.
)trtdoc";
constexpr char const* queryKeys = R"trtdoc(
Query cache keys from Timing Cache.
If an error occurs, a RuntimeError will be raised.
:returns A list containing the cache keys.
)trtdoc";
constexpr char const* query = R"trtdoc(
Query value in a cache entry.
If the key exists, write the value out, otherwise return an invalid value.
:arg key: The query key.
:cache cache: value if the key exists, otherwise an invalid value.
:returns A :class:`TimingCacheValue` object.
)trtdoc";
constexpr char const* update = R"trtdoc(
Update values in a cache entry.
Update the value of the given cache key. If the key does not exist, return False.
If the key exists and the new tactic timing is NaN, delete the cache entry and
return True. If tactic timing is not NaN and the new value is valid, override the
cache value and return True. False is returned when the new value is invalid.
If this layer cannot use the new tactic, build errors will be reported when
building the next engine.
:arg key: The key to the entry to be updated.
:arg value: New cache value.
:returns True if update succeeds, otherwise False.
)trtdoc";
} // namespace ITimingCacheDoc
namespace IBuilderConfigDoc
@@ -1346,13 +1431,15 @@ constexpr char const* descr = R"trtdoc(
:ivar DLA_core: :class:`int` The DLA core that the engine executes on. Must be between 0 and N-1 where N is the number of available DLA cores.
:ivar profiling_verbosity: Profiling verbosity in NVTX annotations.
:ivar engine_capability: The desired engine capability. See :class:`EngineCapability` for details.
:ivar algorithm_selector: The :class:`IAlgorithmSelector` to use.
:ivar algorithm_selector: [DEPRECATED] Deprecated in TensorRT 10.8. Please use editable mode in ITimingCache instead. The :class:`IAlgorithmSelector` to use.
:ivar builder_optimization_level: The builder optimization level which TensorRT should build the engine at. Setting a higher optimization level allows TensorRT to spend longer engine building time searching for more optimization options. The resulting engine may have better performance compared to an engine built with a lower optimization level. The default optimization level is 3. Valid values include integers from 0 to the maximum optimization level, which is currently 5. Setting it to be greater than the maximum level results in identical behavior to the maximum level.
:ivar max_num_tactics: The maximum number of tactics to time when there is a choice of tactics. Setting a larger number allows TensorRT to spend longer engine building time searching for more optimization options. The resulting engine may have better performance compared to an engine built with a smaller number of tactics. Valid values include integers from -1 to the maximum 32-bit integer. Default value -1 indicates that TensorRT can decide the number of tactics based on its own heuristic.
:ivar hardware_compatibility_level: Hardware compatibility allows an engine compatible with GPU architectures other than that of the GPU on which the engine was built.
:ivar plugins_to_serialize: The plugin libraries to be serialized with forward-compatible engines.
:ivar max_aux_streams: The maximum number of auxiliary streams that TRT is allowed to use. If the network contains operators that can run in parallel, TRT can execute them using auxiliary streams in addition to the one provided to the IExecutionContext::enqueueV3() call. The default maximum number of auxiliary streams is determined by the heuristics in TensorRT on whether enabling multi-stream would improve the performance. This behavior can be overridden by calling this API to set the maximum number of auxiliary streams explicitly. Set this to 0 to enforce single-stream inference. The resulting engine may use fewer auxiliary streams than the maximum if the network does not contain enough parallelism or if TensorRT determines that using more auxiliary streams does not help improve the performance. Allowing more auxiliary streams does not always give better performance since there will be synchronizations overhead between streams. Using CUDA graphs at runtime can help reduce the overhead caused by cross-stream synchronizations. Using more auxiliary leads to more memory usage at runtime since some activation memory blocks will not be able to be reused.
:ivar progress_monitor: The :class:`IProgressMonitor` to use.
:ivar tiling_optimization_level: The optimization level of tiling strategies. A Higher level allows TensorRT to spend more time searching for better optimization strategy.
:ivar l2_limit_for_tiling: The target L2 cache usage for tiling optimization.
Below are the descriptions about each builder optimization level:
python/docstrings/infer/pyFoundationalTypesDoc.h
+2
View File
@@ -54,6 +54,8 @@ constexpr char const* fp8 = R"trtdoc(
bits, and exponent-bias 7.
)trtdoc";
constexpr char const* int4 = R"trtdoc(Signed 4-bit integer representing a quantized floating-point value.)trtdoc";
constexpr char const* fp4
= R"trtdoc(Signed 4-bit floating point with 1 sign bit, 2 exponent bits and 1 mantissa bits..)trtdoc";
} // namespace DataTypeDoc
python/docstrings/infer/pyGraphDoc.h
+88 -2
View File
@@ -74,6 +74,8 @@ constexpr char const* NORMALIZATION = R"trtdoc(Normalization layer)trtdoc";
constexpr const char* PLUGIN_V3 = R"trtdoc(PluginV3 layer)trtdoc";
constexpr const char* SQUEEZE = R"trtdoc(Squeeze layer)trtdoc";
constexpr const char* UNSQUEEZE = R"trtdoc(Unsqueeze layer)trtdoc";
constexpr const char* CUMULATIVE = R"trtdoc(Cumulative layer)trtdoc";
constexpr const char* DYNAMIC_QUANTIZE = R"trtdoc(DynamicQuantize layer)trtdoc";
} // namespace LayerTypeDoc
namespace TensorFormatDoc
@@ -170,7 +172,7 @@ constexpr const char* DLA_HWC4 = R"trtdoc(
)trtdoc";
constexpr const char* HWC16 = R"trtdoc(
Sixteen channel format where C is padded to a multiple of 16. This format is bound to FP16 and INT8. It is only available for dimensions >= 3.
Sixteen channel format where C is padded to a multiple of 16. This format is bound to FP16/INT8/FP8. It is only available for dimensions >= 3.
For a tensor with dimensions {N, C, H, W}, the memory layout is equivalent to the array with dimensions [N][H][W][(C+15)/16*16], with the tensor coordinates (n, c, h, w) mapping to array subscript [n][h][w][c].
)trtdoc";
@@ -1658,6 +1660,25 @@ constexpr const char* descr = R"trtdoc(
)trtdoc";
} // namespace IDequantizeLayerDoc
namespace IDynamicQuantizeLayerDoc
{
constexpr const char* descr = R"trtdoc(
A DynamicQuantize layer in an :class:`INetworkDefinition` .
This layer performs dynamic block quantization of its input tensor and outputs the quantized data and the computed block scale-factors.
The block size is currently limited to 16 and the size of the blocked axis must be divisible by 16.
The first input (index 0) is the tensor to be quantized. Its data type must be one of DataType::kFLOAT,
DataType::kHALF, or DataType::kBF16. Currently only 2D and 3D inputs are supported.
The second input (index 1) is the double quantization scale factor. It is a scalar scale factor used to quantize the computed block scales-factors.
:ivar axis: :class:`int` The axis that is sliced into blocks. The axis must be the last dimension or the second to last dimension.
:ivar block_size: :class:`int` The number of elements that are quantized using a shared scale factor. Currently only blocks of 16 elements are supported.
:ivar output_type: :class:`DataType` The data type of the quantized output tensor, must be DataType::kFP4.
:ivar scale_type: :class:`DataType` The data type of the scale factor used for quantizing the input data, must be DataType::kFP8.
)trtdoc";
} // namespace IDynamicQuantizeLayerDoc
namespace IIfConditionalBoundaryLayerDoc
{
@@ -1890,6 +1911,46 @@ constexpr const char* set_input = R"trtdoc(
namespace CumulativeOperationDoc
{
constexpr const char* descr = R"trtdoc(The cumulative operations that may be performed by a Cumulative layer)trtdoc";
constexpr const char* SUM = R"trtdoc()trtdoc";
} // namespace CumulativeOperationDoc
namespace ICumulativeLayerDoc
{
constexpr const char* descr = R"trtdoc(
A cumulative layer in an :class:`INetworkDefinition` .
This layer represents a cumulative operation across a tensor.
It computes successive reductions across an axis of a tensor. The output
always has the same shape as the input.
If the reduction operation is summation, then this is also known as
prefix-sum or cumulative sum.
The operation has forward vs. reverse variants, and inclusive vs. exclusive variants.
For example, let the input be a vector x of length n and the output be vector y.
Then y[j] = sum(x[...]) where ... denotes a sequence of indices from this list:
- inclusive + forward: 0..j
- inclusive + reverse: j..n-1
- exclusive + forward: 0..j-1
- exclusive + reverse: j+1..n-1
For multidimensional tensors, the cumulative applies across a specified axis. For
example, given a 2D input, a forward inclusive cumulative across axis 0 generates
cumulative sums within each column.
:ivar op: :class:`CumulativeOperation` The cumulative operation for the layer.
:ivar exclusive: :class:`bool` Specifies whether it is an exclusive cumulative or inclusive cumulative.
:ivar reverse: :class:`bool` Specifies whether the cumulative operation should be applied backward.
)trtdoc";
} // namespace ICumulativeLayerDoc
namespace INetworkDefinitionDoc
{
constexpr const char* descr = R"trtdoc(
@@ -1916,7 +1977,7 @@ constexpr const char* add_input = R"trtdoc(
Adds an input to the network.
:arg name: The name of the tensor. Each input and output tensor must have a unique name.
:arg dtype: The data type of the tensor. Currently, tensorrt.int8 is not supported for inputs.
:arg dtype: The data type of the tensor.
:arg shape: The dimensions of the tensor. The total volume must be less than 2^31 elements.
:returns: The newly added Tensor.
@@ -2522,6 +2583,18 @@ constexpr const char* add_dequantize = R"trtdoc(
:returns: The new dequantization layer, or :class:`None` if it could not be created.
)trtdoc";
constexpr const char* add_dynamic_quantize = R"trtdoc(
Add a dynamic quantization layer to the network.
See :class:`IDynamicQuantizeLayer` for more information.
:arg input: A tensor to quantize.
:arg axis: The axis that is sliced into blocks.
:arg block_size: The number of elements that are quantized using a shared scale factor.
:arg output_type: The data type of the quantized output tensor.
:arg scale_type: The data type of the scale factor used for quantizing the input data.
:returns: The new DynamicQuantization layer, or :class:`None` if it could not be created.
)trtdoc";
constexpr const char* add_if_conditional = R"trtdoc(
Adds an if-conditional to the network, which provides a way to specify subgraphs that will be conditionally executed using lazy evaluation.
@@ -2596,6 +2669,19 @@ constexpr char const* add_unsqueeze = R"trtdoc(
constexpr const char* add_cumulative = R"trtdoc(
Add a cumulative layer to the network.
See :class:`ICumulativeLayer` for more information.
:arg input: The input tensor to the layer.
:arg axis: The axis tensor to apply the cumulative operation on. Currently, it must be a build-time constant 0-D shape tensor.
:arg op: The reduction operation to perform.
:arg exclusive: The boolean that specifies whether it is an exclusive cumulative or inclusive cumulative.
:arg reverse: The boolean that specifies whether the cumulative should be applied backward.
:returns: The new cumulative layer, or :class:`None` if it could not be created.
)trtdoc";
} // namespace INetworkDefinitionDoc
} // namespace tensorrt