TensorRT OSS 10.1 GA Release

Signed-off-by: Akhil Goel <akhilg@nvidia.com>
2024-06-08 18:25:44 -07:00
parent ccf119972b
commit 21f31ba77e
149 changed files with 5999 additions and 1129 deletions
@@ -397,7 +397,7 @@ constexpr char const* descr = R"trtdoc(
    :ivar profiler: :class:`IProfiler` The profiler in use by this :class:`IExecutionContext` .
    :ivar engine: :class:`ICudaEngine` The associated :class:`ICudaEngine` .
    :ivar name: :class:`str` The name of the :class:`IExecutionContext` .
-    :ivar device_memory: :class:`capsule` The device memory for use by this execution context. The memory must be aligned on a 256-byte boundary, and its size must be at least :attr:`engine.device_memory_size`. If using :func:`execute_v2()`, it is in use until :func:`execute_v2()` returns. Releasing or otherwise using the memory for other purposes during this time will result in undefined behavior.
+    :ivar device_memory: :class:`capsule` The device memory for use by this execution context. The memory must be aligned with cuda memory alignment property (using :func:`cuda.cudart.cudaGetDeviceProperties()`), and its size must be large enough for performing inference with the given network inputs. :func:`engine.device_memory_size` and :func:`engine.get_device_memory_size_for_profile` report upper bounds of the size. Setting memory to nullptr is acceptable if the reported size is 0. If using :func:`execute_async_v3()` to run the network, the memory is in use from the invocation of :func:`execute_async_v3()` until network execution is complete. If using :func:`execute_v2()`, it is in use until :func:`execute_v2()` returns. Releasing or otherwise using the memory for other purposes, including using it in another execution context running in parallel, during this time will result in undefined behavior.
    :ivar active_optimization_profile: :class:`int` The active optimization profile for the context. The selected profile will be used in subsequent calls to :func:`execute_v2()`. Profile 0 is selected by default. This is a readonly property and active optimization profile can be changed with :func:`set_optimization_profile_async()`. Changing this value will invalidate all dynamic bindings for the current execution context, so that they have to be set again using :func:`set_input_shape` before calling either :func:`execute_v2()`.
    :ivar all_binding_shapes_specified: :class:`bool` Whether all dynamic dimensions of input tensors have been specified by calling :func:`set_input_shape` . Trivially true if network has no dynamically shaped input tensors. Does not work with name-base interfaces eg. :func:`set_input_shape()`. Use :func:`infer_shapes()` instead.
    :ivar all_shape_inputs_specified: :class:`bool` Whether values for all input shape tensors have been specified by calling :func:`set_shape_input` . Trivially true if network has no input shape bindings. Does not work with name-base interfaces eg. :func:`set_input_shape()`. Use :func:`infer_shapes()` instead.
@@ -406,9 +406,6 @@ constexpr char const* descr = R"trtdoc(
    :ivar persistent_cache_limit: The maximum size of persistent L2 cache that this execution context may use for activation caching. Activation caching is not supported on all architectures - see "How TensorRT uses Memory" in the developer guide for details. The default is 0 Bytes.
    :ivar nvtx_verbosity: The NVTX verbosity of the execution context. Building with DETAILED verbosity will generally increase latency in enqueueV3(). Call this method to select NVTX verbosity in this execution context at runtime. The default is the verbosity with which the engine was built, and the verbosity may not be raised above that level. This function does not affect how IEngineInspector interacts with the engine.
    :ivar temporary_allocator: :class:`IGpuAllocator` The GPU allocator used for internal temporary storage.
-    :ivar weight_streaming_budget: Set and get the current weight streaming budget for inference. The budget may be set to -1 disabling weight streaming at runtime, 0 (default) enabling TRT to choose to weight stream or not, or a positive value in the inclusive range [minimum_weight_streaming_budget, streamable_weights_size - 1].
-    :ivar minimum_weight_streaming_budget: Returns the minimum weight streaming budget in bytes required to run the network successfully. The engine must have been built with kWEIGHT_STREAMING.
-    :ivar streamable_weights_size: Returns the size of the streamable weights in the engine. This may not include all the weights.
 )trtdoc";

 constexpr char const* execute_v2 = R"trtdoc(
@@ -421,13 +418,14 @@ constexpr char const* execute_v2 = R"trtdoc(
 )trtdoc";

 // TODO: Check if this makes sense to have.
-constexpr char const* device_memory = R"trtdoc(
+constexpr char const* set_device_memory = R"trtdoc(
    The device memory for use by this :class:`IExecutionContext` .

-    The memory must be aligned on a 256-byte boundary, and its size must be at least that
-    returned by getDeviceMemorySize(). If using :func:`execute_v2()`,
-    it is in use until :func:`execute_v2()` returns. Releasing or otherwise using the memory for other
-    purposes during this time will result in undefined behavior.
+    :arg memory: 256-byte aligned device memory.
+    :arg size: Size of the provided memory. This must be at least as large as CudaEngine.get_device_memory_size_v2
+
+    If using :func:`enqueue_v3()`, it is in use until :func:`enqueue_v3()` returns. Releasing or otherwise using the memory for other
+    purposes during this time will result in undefined behavior. This includes using the same memory for a parallel execution context.
 )trtdoc";

 constexpr char const* set_optimization_profile_async = R"trtdoc(
@@ -700,6 +698,7 @@ constexpr char const* descr = R"trtdoc(
    :ivar num_layers: :class:`int` The number of layers in the network. The number of layers in the network is not necessarily the number in the original :class:`INetworkDefinition`, as layers may be combined or eliminated as the :class:`ICudaEngine` is optimized. This value can be useful when building per-layer tables, such as when aggregating profiling data over a number of executions.
    :ivar max_workspace_size: :class:`int` The amount of workspace the :class:`ICudaEngine` uses. The workspace size will be no greater than the value provided to the :class:`Builder` when the :class:`ICudaEngine` was built, and will typically be smaller. Workspace will be allocated for each :class:`IExecutionContext` .
    :ivar device_memory_size: :class:`int` The amount of device memory required by an :class:`IExecutionContext` .
+    :ivar device_memory_size_v2: :class:`int` The amount of device memory required by an :class:`IExecutionContext`. The return value depends on the weight streaming budget if enabled.
    :ivar refittable: :class:`bool` Whether the engine can be refit.
    :ivar name: :class:`str` The name of the network associated with the engine. The name is set during network creation and is retrieved after building or deserialization.
    :ivar num_optimization_profiles: :class:`int` The number of optimization profiles defined for this engine. This is always at least 1.
@@ -708,7 +707,13 @@ constexpr char const* descr = R"trtdoc(
    :ivar tactic_sources: :class:`int` The tactic sources required by this engine.
    :ivar profiling_verbosity: The profiling verbosity the builder config was set to when the engine was built.
    :ivar hardware_compatibility_level: The hardware compatibility level of the engine.
-    :ivar num_aux_streams: Read-only. The number of auxiliary streams used by this engine, which will be less than or equal to the maximum allowed number of auxiliary streams by setting builder_config.max_aux_streams when the engine is built.)trtdoc"
+    :ivar num_aux_streams: Read-only. The number of auxiliary streams used by this engine, which will be less than or equal to the maximum allowed number of auxiliary streams by setting builder_config.max_aux_streams when the engine is built.
+    :ivar weight_streaming_budget: [DEPRECATED] Deprecated in TensorRT 10.1, superceded by weight_streaming_budget_v2. Set and get the current weight streaming budget for inference. The budget may be set to -1 disabling weight streaming at runtime, 0 (default) enabling TRT to choose to weight stream or not, or a positive value in the inclusive range [minimum_weight_streaming_budget, streamable_weights_size - 1].
+    :ivar minimum_weight_streaming_budget: [DEPRECATED] Deprecated in TensorRT 10.1, superceded by weight_streaming_budget_v2. Returns the minimum weight streaming budget in bytes required to run the network successfully. The engine must have been built with kWEIGHT_STREAMING.
+    :ivar streamable_weights_size: Returns the size of the streamable weights in the engine. This may not include all the weights.
+    :ivar weight_streaming_budget_v2: Set and get the current weight streaming budget for inference. The budget may be set any non-negative value. A value of 0 streams the most weights. Values equal to streamable_weights_size (default) or larger will disable weight streaming.
+    :ivar weight_streaming_scratch_memory_size: The amount of scratch memory required by a TensorRT ExecutionContext to perform inference. This value may change based on the current weight streaming budget. Please use the V2 memory APIs, engine.device_memory_size_v2 and ExecutionContext.set_device_memory() to provide memory which includes the current weight streaming scratch memory. Not specifying these APIs or using the V1 APIs will not include this memory, so TensorRT will resort to allocating itself.
+    )trtdoc"
           ;

 // Documentation bug with parameters on these three functions because they are overloaded.
@@ -841,6 +846,15 @@ constexpr char const* get_device_memory_size_for_profile = R"trtdoc(
    :arg profile_index: The index of the profile.
 )trtdoc";

+constexpr char const* get_device_memory_size_for_profile_v2 = R"trtdoc(
+    Return the device memory size required for a certain profile.
+
+    The return value will change depending on the following API calls
+    1. setWeightStreamingBudgetV2
+
+    :arg profile_index: The index of the profile.
+)trtdoc";
+
 constexpr char const* create_serialization_config = R"trtdoc(
    Create a serialization configuration object.
 )trtdoc";
@@ -849,6 +863,11 @@ constexpr char const* serialize_with_config = R"trtdoc(
    Serialize the network to a stream.
 )trtdoc";

+constexpr char const* get_weight_streaming_automatic_budget = R"trtdoc(
+    Get an automatic weight streaming budget based on available device memory. This value may change between TensorRT major and minor versions.
+    Please use CudaEngine.weight_streaming_budget_v2 to set the returned budget.
+)trtdoc";
+
 constexpr char const* is_debug_tensor = R"trtdoc(
    Determine whether the given name corresponds to a debug tensor.

@@ -975,7 +994,7 @@ constexpr char const* REJECT_EMPTY_ALGORITHMS
 constexpr char const* VERSION_COMPATIBLE
    = R"trtdoc(Restrict to lean runtime operators to provide version forward compatibility for the plan files.)trtdoc";
 constexpr char const* EXCLUDE_LEAN_RUNTIME = R"trtdoc(Exclude lean runtime from the plan.)trtdoc";
-constexpr char const* FP8 = R"trtdoc(Enable FP8 layer selection)trtdoc";
+constexpr char const* FP8 = R"trtdoc(Enable plugins with FP8 input/output)trtdoc";
 constexpr char const* ERROR_ON_TIMING_CACHE_MISS
    = R"trtdoc(Emit error when a tactic being timed is not present in the timing cache.)trtdoc";
 constexpr char const* DISABLE_COMPILATION_CACHE
@@ -989,6 +1008,7 @@ constexpr char const* REFIT_IDENTICAL
    = R"trtdoc(Create a refittable engine using identical weights. Different weights during refits yield unpredictable behavior.)trtdoc";
 constexpr char const* WEIGHT_STREAMING
    = R"trtdoc(Enable building with the ability to stream varying amounts of weights during Runtime. This decreases GPU memory of TRT at the expense of performance.)trtdoc";
+constexpr char const* INT4 = R"trtdoc(Enable plugins with INT4 input/output)trtdoc";
 } // namespace BuilderFlagDoc

 namespace MemoryPoolTypeDoc
@@ -1153,7 +1173,7 @@ constexpr char const* CUBLAS = R"trtdoc(
        **NOTE:** Disabling CUBLAS tactic source will cause the cuBLAS handle passed to plugins in attachToContext to be null.
    )trtdoc";
 constexpr char const* CUBLAS_LT = R"trtdoc(
-        Enables CUBLAS_LT tactics. Disabled by default.
+        Enables cuBLAS LT tactics. Disabled by default.
        [DEPRECATED] Deprecated in TensorRT 9.0.
    )trtdoc";
 constexpr char const* CUDNN = R"trtdoc(
@@ -1240,7 +1260,7 @@ namespace IBuilderConfigDoc
 constexpr char const* descr = R"trtdoc(

        :ivar avg_timing_iterations: :class:`int` The number of averaging iterations used when timing layers. When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations used in averaging. By default the number of averaging iterations is 1.
-        :ivar int8_calibrator: :class:`IInt8Calibrator` Int8 Calibration interface. The calibrator is to minimize the information loss during the INT8 quantization process.
+        :ivar int8_calibrator: :class:`IInt8Calibrator` [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization. Int8 Calibration interface. The calibrator is to minimize the information loss during the INT8 quantization process.
        :ivar flags: :class:`int` The build mode flags to turn on builder options for this network. The flags are listed in the BuilderFlags enum. The flags set configuration options to build the network. This should be in integer consisting of one or more :class:`BuilderFlag` s, combined via binary OR. For example, ``1 << BuilderFlag.FP16 | 1 << BuilderFlag.DEBUG``.
        :ivar profile_stream: :class:`int` The handle for the CUDA stream that is used to profile this network.
        :ivar num_optimization_profiles: :class:`int` The number of optimization profiles.
@@ -1353,6 +1373,8 @@ constexpr char const* add_optimization_profile = R"trtdoc(
 )trtdoc";

 constexpr char const* set_calibration_profile = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Set a calibration profile.

    Calibration optimization profile must be set if int8 calibration is used to set scales for a network with runtime dimensions.
@@ -1363,6 +1385,8 @@ constexpr char const* set_calibration_profile = R"trtdoc(
 )trtdoc";

 constexpr char const* get_calibration_profile = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Get the current calibration profile.

    :returns: The current calibration profile or None if calibrartion profile is unset.
@@ -1866,6 +1890,8 @@ constexpr char const* get_all_weights = R"trtdoc(
 )trtdoc";

 constexpr char const* get_dynamic_range = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Gets the dynamic range of a tensor. If the dynamic range was never set, returns the range computed during calibration.

    :arg tensor_name: The name of the tensor whose dynamic range to retrieve.
@@ -1874,6 +1900,8 @@ constexpr char const* get_dynamic_range = R"trtdoc(
 )trtdoc";

 constexpr char const* set_dynamic_range = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Update dynamic range for a tensor.

    :arg tensor_name: The name of the tensor whose dynamic range to update.
@@ -1885,6 +1913,8 @@ constexpr char const* set_dynamic_range = R"trtdoc(
 )trtdoc";

 constexpr char const* get_tensors_with_dynamic_range = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Get names of all tensors that have refittable dynamic ranges.

    :returns: The names of tensors with refittable dynamic ranges.
@@ -52,9 +52,6 @@ constexpr char const* uint8 = R"trtdoc(
 constexpr char const* fp8 = R"trtdoc(
    Signed 8-bit floating point with 1 sign bit, 4 exponent bits, 3 mantissa
    bits, and exponent-bias 7.
-
-    .. warning::
-       fp8 is not supported yet and will result in an error or undefined behavior.
 )trtdoc";
 constexpr char const* int4 = R"trtdoc(Signed 4-bit integer representing a quantized floating-point value.)trtdoc";

@@ -195,12 +195,13 @@ constexpr const char* descr = R"trtdoc(
    :ivar location: :class:`TensorLocation` The storage location of a tensor.
    :ivar is_network_input: :class:`bool` Whether the tensor is a network input.
    :ivar is_network_output: :class:`bool` Whether the tensor is a network output.
-    :ivar dynamic_range: :class:`Tuple[float, float]` A tuple containing the [minimum, maximum] of the dynamic range, or :class:`None` if the range was not set.
+    :ivar dynamic_range: :class:`Tuple[float, float]` [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization. A tuple containing the [minimum, maximum] of the dynamic range, or :class:`None` if the range was not set.
    :ivar is_shape: :class:`bool` Whether the tensor is a shape tensor.
    :ivar allowed_formats: :class:`int32` The allowed set of TensorFormat candidates. This should be an integer consisting of one or more :class:`TensorFormat` s, combined via bitwise OR after bit shifting. For example, ``1 << int(TensorFormat.CHW4) | 1 << int(TensorFormat.CHW32)``.
 )trtdoc"
        ;
 constexpr const char* set_dynamic_range = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
    Set dynamic range for the tensor.
    NOTE: It is suggested to use ``tensor.dynamic_range = (min, max)`` instead.

@@ -210,6 +211,7 @@ constexpr const char* set_dynamic_range = R"trtdoc(
 )trtdoc";

 constexpr const char* get_dynamic_range = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
    Get dynamic range for the tensor.
    NOTE: It is suggested to use ``tensor.dynamic_range`` instead, which is a tuple including both the minimum and maximum of the dynamic range.

@@ -217,6 +219,7 @@ constexpr const char* get_dynamic_range = R"trtdoc(
 )trtdoc";

 constexpr const char* reset_dynamic_range = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
    Undo the effect of setting the dynamic range.
 )trtdoc";

@@ -663,6 +666,8 @@ constexpr const char* SIGN
 constexpr const char* ROUND = R"trtdoc(Round to nearest even for floating-point data type.)trtdoc";
 constexpr const char* ISINF
    = R"trtdoc(Return true if the input value equals +/- infinity for floating-point data type.)trtdoc";
+constexpr const char* ISNAN
+    = R"trtdoc(Return true if the input value equals NaN for floating-point data type.)trtdoc";
 } // namespace UnaryOperationDoc

 namespace IUnaryLayerDoc
@@ -848,7 +853,7 @@ constexpr const char* descr = R"trtdoc(
    This class sets the output to a one-dimensional tensor with the dimensions of the input tensor.

    For example, if the input is a four-dimensional tensor (of any type) with
-    dimensions [2,3,5,7], the output tensor is a one-dimensional :class:`int32` tensor
+    dimensions [2,3,5,7], the output tensor is a one-dimensional :class:`int64` tensor
    of length 4 containing the sequence 2, 3, 5, 7.
 )trtdoc";

@@ -31,6 +31,8 @@ constexpr const char* descr = R"trtdoc(
 namespace IInt8CalibratorDoc
 {
 constexpr const char* descr = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Application-implemented interface for calibration. Calibration is a step performed by the builder when deciding suitable scale factors for 8-bit inference. It must also provide a method for retrieving representative images which the calibration process can use to examine the distribution of activations. It may optionally implement a method for caching the calibration result for reuse on subsequent runs.

    To implement a custom calibrator, ensure that you explicitly instantiate the base class in :func:`__init__` :
@@ -117,6 +119,8 @@ constexpr const char* write_calibration_cache = R"trtdoc(
 namespace IInt8LegacyCalibratorDoc
 {
 constexpr const char* descr = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Extends the :class:`IInt8Calibrator` class.
    This calibrator requires user parameterization, and is provided as a fallback option if the other calibrators yield poor results.

@@ -160,6 +164,8 @@ constexpr const char* get_algorithm = R"trtdoc(
 namespace IInt8EntropyCalibratorDoc
 {
 constexpr const char* descr = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Extends the :class:`IInt8Calibrator` class.

    To implement a custom calibrator, ensure that you explicitly instantiate the base class in :func:`__init__` :
@@ -183,6 +189,8 @@ constexpr const char* get_algorithm = R"trtdoc(
 namespace IInt8EntropyCalibrator2Doc
 {
 constexpr const char* descr = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Extends the :class:`IInt8Calibrator` class.

    To implement a custom calibrator, ensure that you explicitly instantiate the base class in :func:`__init__` :
@@ -205,6 +213,8 @@ constexpr const char* get_algorithm = R"trtdoc(
 namespace IInt8MinMaxCalibratorDoc
 {
 constexpr const char* descr = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. Superseded by explicit quantization.
+
    Extends the :class:`IInt8Calibrator` class.

    To implement a custom calibrator, ensure that you explicitly instantiate the base class in :func:`__init__` :
@@ -59,6 +59,8 @@ constexpr const char* parse_from_file = R"trtdoc(
 )trtdoc";

 constexpr const char* supports_model = R"trtdoc(
+    [DEPRECATED] Deprecated in TensorRT 10.1. See supports_model_v2.
+
    Check whether TensorRT supports a particular ONNX model.

    :arg model: The serialized ONNX model.
@@ -69,6 +71,38 @@ constexpr const char* supports_model = R"trtdoc(
        The second indicates subgraphs (by node index) in the model and whether they are supported.
 )trtdoc";

+constexpr const char* supports_model_v2 = R"trtdoc(
+    Check whether TensorRT supports a particular ONNX model.
+    Query each subgraph with num_subgraphs, is_subgraph_supported, get_subgraph_nodes.
+
+    :arg model: The serialized ONNX model.
+    :arg path: The path to the model file. Only required if the model has externally stored weights.
+    :returns: true if the model is supported
+)trtdoc";
+
+constexpr const char* num_subgraphs = R"trtdoc(
+    Get the number of subgraphs. Calling before \p supportsModelV2 is an undefined behavior. Will return 0 by default.
+
+    :returns: Number of subgraphs
+)trtdoc";
+
+constexpr const char* is_subgraph_supported = R"trtdoc(
+    Returns whether the subgraph is supported. Calling before \p supportsModelV2 is an undefined behavior.
+    Will return false by default.
+
+    :arg index: Index of the subgraph to be checked.
+    :returns: true if subgraph is supported
+)trtdoc";
+
+constexpr const char* get_subgraph_nodes = R"trtdoc(
+    Get the nodes of the specified subgraph. Calling before \p supportsModelV2 is an undefined behavior.
+    Will return an empty list by default.
+
+    :arg index: Index of the subgraph.
+    :returns: List[int]
+        A list of node indices in the subgraph.
+)trtdoc";
+
 constexpr const char* supports_operator = R"trtdoc(
    Returns whether the specified operator may be supported by the parser.
    Note that a result of true does not guarantee that the operator will be supported in all cases (i.e., this function may return false-positives).
@@ -20,6 +20,7 @@ import os
 import sys
 import warnings

+ENABLE_LONG_TERM = bool(int(os.environ.get("NV_INTERNAL_ENABLE_LONG_TERM", "0")))

 # For standalone wheels, attempt to import the wheel containing the libraries.
 _libs_wheel_imported = False
@@ -196,6 +197,10 @@ def _itemsize(trt_type):
        fp8: 1,
        int4: 0.5,
    }
+    # $nv-internal-release begin
+    if ENABLE_LONG_TERM:
+        mapping[fp4] = 0.5
+    # $nv-internal-release end
    if trt_type in mapping:
        return mapping[trt_type]

@@ -157,7 +157,7 @@ When the extra index url does not contain `{}`, a nested `pip install` will run
    ],
    packages=[tensorrt_package],
    install_requires=install_requires,
-    setup_requires=["wheel"],
+    setup_requires=["wheel", "pip"],
    python_requires=">=3.6",  # ref https://pypi.nvidia.com/tensorrt-bindings/
    cmdclass=cmdclass,
    extras_require={"numpy": "numpy"},
@@ -336,6 +336,11 @@ void context_set_device_memory(IExecutionContext& self, size_t memory)
    self.setDeviceMemory(reinterpret_cast<void*>(memory));
 }

+void context_set_device_memory_v2(IExecutionContext& self, size_t memory, int64_t size)
+{
+    self.setDeviceMemoryV2(reinterpret_cast<void*>(memory), size);
+}
+
 void serialization_config_set_flags(ISerializationConfig& self, uint32_t flags)
 {
    if (!self.setFlags(flags))
@@ -1034,6 +1039,8 @@ void bindCore(py::module& m)
            "name", &IExecutionContext::getName, py::cpp_function(&IExecutionContext::setName, py::keep_alive<1, 2>{}))
        // For writeonly properties, we use a nullptr getter.
        .def_property("device_memory", nullptr, &lambdas::context_set_device_memory)
+        .def("set_device_memory", &lambdas::context_set_device_memory_v2, "memory"_a, "size"_a,
+            IExecutionContextDoc::set_device_memory)
        .def("update_device_memory_size_for_shapes", &IExecutionContext::updateDeviceMemorySizeForShapes,
            IExecutionContextDoc::update_device_memory_size_for_shapes)
        .def_property_readonly("active_optimization_profile", &IExecutionContext::getOptimizationProfile)
@@ -1162,9 +1169,16 @@ void bindCore(py::module& m)
            utils::deprecateMember(&ICudaEngine::createExecutionContextWithoutDeviceMemory, "create_execution_context"),
            ICudaEngineDoc::create_execution_context_without_device_memory, py::keep_alive<0, 1>{},
            py::call_guard<py::gil_scoped_release>{})
-        .def("get_device_memory_size_for_profile", &ICudaEngine::getDeviceMemorySizeForProfile, "profile_index"_a,
-            ICudaEngineDoc::get_device_memory_size_for_profile)
-        .def_property_readonly("device_memory_size", &ICudaEngine::getDeviceMemorySize)
+        .def("get_device_memory_size_for_profile",
+            utils::deprecateMember(&ICudaEngine::getDeviceMemorySizeForProfile,
+                "Deprecated in TensorRT 10.1. Superseded by get_device_memory_size_for_profile_v2"),
+            "profile_index"_a, ICudaEngineDoc::get_device_memory_size_for_profile)
+        .def("get_device_memory_size_for_profile_v2", &ICudaEngine::getDeviceMemorySizeForProfile, "profile_index"_a,
+            ICudaEngineDoc::get_device_memory_size_for_profile_v2)
+        .def_property_readonly("device_memory_size",
+            utils::deprecateMember(&ICudaEngine::getDeviceMemorySize,
+                "Deprecated in TensorRT 10.1. Superseded by get_device_memory_size_v2"))
+        .def_property_readonly("device_memory_size_v2", &ICudaEngine::getDeviceMemorySizeV2)
        .def_property_readonly("refittable", &ICudaEngine::isRefittable)
        .def_property_readonly("name", &ICudaEngine::getName)
        .def_property_readonly("num_optimization_profiles", &ICudaEngine::getNbOptimizationProfiles)
@@ -1258,10 +1272,26 @@ void bindCore(py::module& m)
        .def_property_readonly("hardware_compatibility_level", &ICudaEngine::getHardwareCompatibilityLevel)
        .def_property_readonly("num_aux_streams", &ICudaEngine::getNbAuxStreams)
        // Weight streaming APIs
-        .def_property(
-            "weight_streaming_budget", &ICudaEngine::getWeightStreamingBudget, &ICudaEngine::setWeightStreamingBudget)
-        .def_property_readonly("minimum_weight_streaming_budget", &ICudaEngine::getMinimumWeightStreamingBudget)
+        .def_property("weight_streaming_budget",
+            utils::deprecateMember(&ICudaEngine::getWeightStreamingBudget,
+                "Deprecated in TensorRT 10.1. Superseded by weight_streaming_budget_v2"),
+            utils::deprecateMember(&ICudaEngine::setWeightStreamingBudget,
+                "Deprecated in TensorRT 10.1. Superseded by weight_streaming_budget_v2"))
+        .def_property("weight_streaming_budget_v2", &ICudaEngine::getWeightStreamingBudgetV2,
+            &ICudaEngine::setWeightStreamingBudgetV2)
+        .def_property_readonly("minimum_weight_streaming_budget",
+            utils::deprecateMember(
+                &ICudaEngine::getMinimumWeightStreamingBudget, "Deprecated in TensorRT 10.1. Not required by V2 APIs."))
        .def_property_readonly("streamable_weights_size", &ICudaEngine::getStreamableWeightsSize)
+        // We keep this as a method so that future TRT versions may overload if the automatic budgeting algorithm ever
+        // requires additional arguments.
+        .def(
+            "get_weight_streaming_automatic_budget",
+            [](ICudaEngine& self) -> int32_t { return self.getWeightStreamingAutomaticBudget(); },
+            ICudaEngineDoc::get_weight_streaming_automatic_budget)
+        .def_property_readonly(
+            "weight_streaming_scratch_memory_size", &ICudaEngine::getWeightStreamingScratchMemorySize)
+        // End weight streaming APIs
        .def("is_debug_tensor", &ICudaEngine::isDebugTensor, "name"_a, ICudaEngineDoc::is_debug_tensor)
               .def("__del__", &utils::doNothingDel<ICudaEngine>);

@@ -1332,7 +1362,9 @@ void bindCore(py::module& m)
        .value("WEIGHTLESS", BuilderFlag::kWEIGHTLESS, BuilderFlagDoc::WEIGHTLESS)
        .value("STRIP_PLAN", BuilderFlag::kSTRIP_PLAN, BuilderFlagDoc::STRIP_PLAN)
        .value("REFIT_IDENTICAL", BuilderFlag::kREFIT_IDENTICAL, BuilderFlagDoc::REFIT_IDENTICAL)
-        .value("WEIGHT_STREAMING", BuilderFlag::kWEIGHT_STREAMING, BuilderFlagDoc::WEIGHT_STREAMING);
+        .value("WEIGHT_STREAMING", BuilderFlag::kWEIGHT_STREAMING, BuilderFlagDoc::WEIGHT_STREAMING)
+        .value("INT4", BuilderFlag::kINT4, BuilderFlagDoc::INT4)
+        ;

    py::enum_<MemoryPoolType>(m, "MemoryPoolType", MemoryPoolTypeDoc::descr, py::module_local())
        .value("WORKSPACE", MemoryPoolType::kWORKSPACE, MemoryPoolTypeDoc::WORKSPACE)
@@ -1386,8 +1418,12 @@ void bindCore(py::module& m)
    py::class_<IBuilderConfig>(m, "IBuilderConfig", IBuilderConfigDoc::descr, py::module_local())
        .def_property(
            "avg_timing_iterations", &IBuilderConfig::getAvgTimingIterations, &IBuilderConfig::setAvgTimingIterations)
-        .def_property("int8_calibrator", &IBuilderConfig::getInt8Calibrator,
-            py::cpp_function(&IBuilderConfig::setInt8Calibrator, py::keep_alive<1, 2>{}))
+        .def_property("int8_calibrator",
+            utils::deprecateMember(&IBuilderConfig::getInt8Calibrator,
+                "Deprecated in TensorRT 10.1. Superseded by explicit quantization."),
+            py::cpp_function(utils::deprecateMember(&IBuilderConfig::setInt8Calibrator,
+                                 "Deprecated in TensorRT 10.1. Superseded by explicit quantization."),
+                py::keep_alive<1, 2>{}))
        .def_property("engine_capability", &IBuilderConfig::getEngineCapability, &IBuilderConfig::setEngineCapability)
        .def("set_memory_pool_limit", &IBuilderConfig::setMemoryPoolLimit, "pool"_a, "pool_size"_a,
            IBuilderConfigDoc::set_memory_pool_limit)
@@ -1412,9 +1448,13 @@ void bindCore(py::module& m)
        .def_property("profile_stream", lambdas::netconfig_get_profile_stream, lambdas::netconfig_set_profile_stream)
        .def("add_optimization_profile", &IBuilderConfig::addOptimizationProfile, "profile"_a,
            IBuilderConfigDoc::add_optimization_profile)
-        .def("set_calibration_profile", &IBuilderConfig::setCalibrationProfile, "profile"_a,
-            IBuilderConfigDoc::set_calibration_profile)
-        .def("get_calibration_profile", &IBuilderConfig::getCalibrationProfile,
+        .def("set_calibration_profile",
+            utils::deprecateMember(&IBuilderConfig::setCalibrationProfile,
+                "Deprecated in TensorRT 10.1. Superseded by explicit quantization."),
+            "profile"_a, IBuilderConfigDoc::set_calibration_profile)
+        .def("get_calibration_profile",
+            utils::deprecateMember(&IBuilderConfig::getCalibrationProfile,
+                "Deprecated in TensorRT 10.1. Superseded by explicit quantization."),
            IBuilderConfigDoc::get_calibration_profile)
        .def_property_readonly("num_optimization_profiles", &IBuilderConfig::getNbOptimizationProfiles)
        .def("set_device_type", &IBuilderConfig::setDeviceType, "layer"_a, "device_type"_a,
@@ -1523,10 +1563,18 @@ void bindCore(py::module& m)
        .def("get_missing_weights", lambdas::refitter_get_missing_weights, RefitterDoc::get_missing_weights)
        .def("get_all", lambdas::refitter_get_all, RefitterDoc::get_all)
        .def("get_all_weights", lambdas::refitter_get_all_weights, RefitterDoc::get_all_weights)
-        .def("get_dynamic_range", lambdas::refitter_get_dynamic_range, "tensor_name"_a, RefitterDoc::get_dynamic_range)
-        .def("set_dynamic_range", lambdas::refitter_set_dynamic_range, "tensor_name"_a, "range"_a,
-            RefitterDoc::set_dynamic_range)
-        .def("get_tensors_with_dynamic_range", lambdas::refitter_get_tensors_with_dynamic_range,
+        // Using a plus sign converts the lambda function into a function pointer.
+        .def("get_dynamic_range",
+            utils::deprecate(+lambdas::refitter_get_dynamic_range,
+                "Deprecated in TensorRT 10.1. Superseded by explicit quantization."),
+            "tensor_name"_a, RefitterDoc::get_dynamic_range)
+        .def("set_dynamic_range",
+            utils::deprecate(+lambdas::refitter_set_dynamic_range,
+                "Deprecated in TensorRT 10.1. Superseded by explicit quantization."),
+            "tensor_name"_a, "range"_a, RefitterDoc::set_dynamic_range)
+        .def("get_tensors_with_dynamic_range",
+            utils::deprecate(+lambdas::refitter_get_tensors_with_dynamic_range,
+                "Deprecated in TensorRT 10.1. Superseded by explicit quantization."),
            RefitterDoc::get_tensors_with_dynamic_range)
        .def_property("error_recorder", &IRefitter::getErrorRecorder,
            py::cpp_function(&IRefitter::setErrorRecorder, py::keep_alive<1, 2>{}))
@@ -186,7 +186,8 @@ void bindFoundationalTypes(py::module& m)
        .value("BOOL", DataType::kBOOL, DataTypeDoc::boolean)
        .value("UINT8", DataType::kUINT8, DataTypeDoc::uint8)
        .value("FP8", DataType::kFP8, DataTypeDoc::fp8)
-        .value("INT4", DataType::kINT4, DataTypeDoc::int4); // DataType
+        .value("INT4", DataType::kINT4, DataTypeDoc::int4)
+        ; // DataType

    // Also create direct mappings (so we can call trt.float32, for example).
    m.attr("float32") = DataType::kFLOAT;
@@ -327,10 +327,11 @@ namespace tensorrt
            .def_property_readonly("is_network_output", &ITensor::isNetworkOutput)
            .def_property_readonly("is_shape_tensor", &ITensor::isShapeTensor)
            .def_property_readonly("is_execution_tensor", &ITensor::isExecutionTensor)
-            .def_property("dynamic_range", lambdas::get_dynamic_range, lambdas::set_dynamic_range)
+            // Using a plus sign converts the lambda function into a function pointer.
+            .def_property("dynamic_range", utils::deprecate(+lambdas::get_dynamic_range, "Deprecated in TensorRT 10.1. Superseded by explicit quantization."), utils::deprecate(+lambdas::set_dynamic_range, "Deprecated in TensorRT 10.1. Superseded by explicit quantization."))
            .def_property("allowed_formats", &ITensor::getAllowedFormats, &ITensor::setAllowedFormats)
-            .def("set_dynamic_range", &ITensor::setDynamicRange, "min"_a, "max"_a, ITensorDoc::set_dynamic_range)
-            .def("reset_dynamic_range", &ITensor::resetDynamicRange, ITensorDoc::reset_dynamic_range)
+            .def("set_dynamic_range", utils::deprecateMember(&ITensor::setDynamicRange, "Deprecated in TensorRT 10.1. Superseded by explicit quantization."), "min"_a, "max"_a, ITensorDoc::set_dynamic_range)
+            .def("reset_dynamic_range", utils::deprecateMember(&ITensor::resetDynamicRange, "Deprecated in TensorRT 10.1. Superseded by explicit quantization."), ITensorDoc::reset_dynamic_range)
            .def("set_dimension_name", &ITensor::setDimensionName, "index"_a, "name"_a, ITensorDoc::set_dimension_name)
            .def("get_dimension_name", &ITensor::getDimensionName, "index"_a, ITensorDoc::get_dimension_name)
        ;
@@ -549,6 +550,7 @@ namespace tensorrt
            .value("SIGN", UnaryOperation::kSIGN, UnaryOperationDoc::SIGN)
            .value("ROUND", UnaryOperation::kROUND, UnaryOperationDoc::ROUND)
            .value("ISINF", UnaryOperation::kISINF, UnaryOperationDoc::ISINF)
+            .value("ISNAN", UnaryOperation::kISNAN, UnaryOperationDoc::ISNAN)
        ;

        py::class_<IUnaryLayer, ILayer, std::unique_ptr<IUnaryLayer, py::nodelete>>(m, "IUnaryLayer", IUnaryLayerDoc::descr, py::module_local())
@@ -975,6 +975,8 @@ public:
                py::function pyGetValidTactics
                    = py::get_override(static_cast<IPluginV3OneBuild const*>(this), "get_valid_tactics");

+                mIsTacticsInitialized = true;
+
                if (!pyGetValidTactics)
                {
                    // if no implementation is provided for get_valid_tactics(), communicate that no custom tactics are
@@ -983,8 +985,8 @@ public:
                }

                py::object pyResult = pyGetValidTactics();
-                auto result = pyResult.cast<std::vector<int32_t>>();
-                return static_cast<int32_t>(result.size());
+                mTactics = pyResult.cast<std::vector<int32_t>>();
+                return static_cast<int32_t>(mTactics.size());
            }
            PLUGIN_API_CATCH_CAST("get_valid_tactics", "std::vector<int32_t>")
            catch (py::error_already_set& e)
@@ -1004,19 +1006,28 @@ public:

            try
            {
-                py::function pyGetValidTactics
-                    = py::get_override(static_cast<IPluginV3OneBuild const*>(this), "get_valid_tactics");
-
-                if (!pyGetValidTactics)
+                // getValidTactics() must immediately follow getNbTactics()
+                // because it is impossible to call getValidTactics() without knowing the
+                // correct number of tactics. So check that mIsTacticsInitialized is true.
+                // Otherwise, something has gone wrong.
+                if (mIsTacticsInitialized)
                {
-                    // if no implementation is provided for get_valid_tactics() nothing further to do
+                    // Unset to catch any subsequent violations
+                    mIsTacticsInitialized = false;
+                    if (nbTactics != static_cast<int32_t>(mTactics.size()))
+                    {
+                        utils::throwPyError(
+                            PyExc_RuntimeError, "number of tactics does not match cached number of tactics");
+                    }
+                    std::copy(mTactics.begin(), mTactics.end(), tactics);
                    return 0;
                }
-
-                py::object pyResult = pyGetValidTactics();
-                auto result = pyResult.cast<std::vector<int32_t>>();
-                std::copy(result.begin(), result.end(), tactics);
-                return 0;
+                else
+                {
+                    utils::throwPyError(
+                        PyExc_RuntimeError, "Internal error. getValidTactics() called before getNbTactics().");
+                }
+                return -1;
            }
            PLUGIN_API_CATCH_CAST("get_valid_tactics", "std::vector<int32_t>")
            catch (py::error_already_set& e)
@@ -1313,11 +1324,13 @@ private:
    int32_t mFormatCombinationLimit{};
    std::string mTimingCachedId{};
    std::string mMetadataString{};
+    std::vector<int32_t> mTactics;

    bool mIsNbOutputsInitialized{false};
    bool mIsTimingCachedIdInitialized{false};
    bool mIsFormatCombinationLimitInitialized{false};
    bool mIsMetadataStringInitialized{false};
+    bool mIsTacticsInitialized{false};
 };

 class PyIPluginV3OneRuntimeImpl : public IPluginV3OneRuntime
@@ -2322,7 +2335,9 @@ void bindPlugin(py::module& m)
        .value("UNKNOWN", PluginFieldType::kUNKNOWN)
        .value("BF16", PluginFieldType::kBF16)
        .value("INT64", PluginFieldType::kINT64)
-        .value("FP8", PluginFieldType::kFP8);
+        .value("FP8", PluginFieldType::kFP8)
+        .value("INT4", PluginFieldType::kINT4)
+        ;

    py::class_<PluginField>(m, "PluginField", PluginFieldDoc::descr, py::module_local())
        .def(py::init(lambdas::plugin_field_default_constructor), "name"_a = "", py::keep_alive<1, 2>{})
@@ -2337,27 +2352,23 @@ void bindPlugin(py::module& m)
            [](PluginField& self) {
                switch (self.type)
                {
-                case PluginFieldType::kINT32:
-                    return py::array(self.length, static_cast<int32_t const*>(self.data));
-                    break;
-                case PluginFieldType::kINT8:
-                    return py::array(self.length, static_cast<int8_t const*>(self.data));
-                    break;
-                case PluginFieldType::kINT16:
-                    return py::array(self.length, static_cast<int16_t const*>(self.data));
-                    break;
+                case PluginFieldType::kINT32: return py::array(self.length, static_cast<int32_t const*>(self.data));
+                case PluginFieldType::kUNKNOWN:
+                case PluginFieldType::kINT8: return py::array(self.length, static_cast<int8_t const*>(self.data));
+                case PluginFieldType::kINT16: return py::array(self.length, static_cast<int16_t const*>(self.data));
+                case PluginFieldType::kFLOAT32: return py::array(self.length, static_cast<float const*>(self.data));
+                case PluginFieldType::kFLOAT64: return py::array(self.length, static_cast<double const*>(self.data));
+                case PluginFieldType::kINT64: return py::array(self.length, static_cast<int64_t const*>(self.data));
+                case PluginFieldType::kCHAR: return py::array(self.length, static_cast<char const*>(self.data));
+                case PluginFieldType::kINT4:
                case PluginFieldType::kFLOAT16:
-                    // TODO: Figure out how to handle float16 correctly here
-                    return py::array(self.length, static_cast<float const*>(self.data));
+                case PluginFieldType::kBF16:
+                case PluginFieldType::kDIMS:
+                case PluginFieldType::kFP8:
+                    utils::throwPyError(
+                        PyExc_AttributeError, "No known conversion for returning data from PluginField");
                    break;
-                case PluginFieldType::kFLOAT32:
-                    return py::array(self.length, static_cast<float const*>(self.data));
-                    break;
-                case PluginFieldType::kFLOAT64:
-                    return py::array(self.length, static_cast<double const*>(self.data));
-                    break;
-                case PluginFieldType::kCHAR: return py::array(self.length, static_cast<char const*>(self.data)); break;
-                default: assert(false && "No known conversion for returning data from PluginField"); break;
+                default: return py::array();
                }
                // should not reach this line
                return py::array();
@@ -55,9 +55,9 @@ static const auto error_code_str = [](ErrorCode self) {
 };

 static const auto parser_error_str = [](IParserError& self) {
-    const std::string node_info = "In node " + std::to_string(self.node()) + " with name: " + self.nodeName()
+    std::string const node_info = "In node " + std::to_string(self.node()) + " with name: " + self.nodeName()
        + " and operator: " + self.nodeOperator() + " ";
-    const std::string error_info
+    std::string const error_info
        = std::string("(") + self.func() + "): " + error_code_str(self.code()) + ": " + self.desc();
    if (self.code() == ErrorCode::kMODEL_DESERIALIZE_FAILED || self.code() == ErrorCode::kREFIT_FAILED)
    {
@@ -66,26 +66,45 @@ static const auto parser_error_str = [](IParserError& self) {
    return node_info + error_info;
 };

-static const auto parse = [](IParser& self, const py::buffer& model, const char* path = nullptr) {
+static const auto parse = [](IParser& self, py::buffer const& model, char const* path = nullptr) {
    py::buffer_info info = model.request();
    return self.parse(info.ptr, info.size * info.itemsize, path);
 };

-static const auto parse_with_weight_descriptors = [](IParser& self, const py::buffer& model) {
+static const auto parse_with_weight_descriptors = [](IParser& self, py::buffer const& model) {
    py::buffer_info info = model.request();
    return self.parseWithWeightDescriptors(info.ptr, info.size * info.itemsize);
 };

 static const auto parseFromFile
-    = [](IParser& self, const std::string& model) { return self.parseFromFile(model.c_str(), 0); };
+    = [](IParser& self, std::string const& model) { return self.parseFromFile(model.c_str(), 0); };

-static const auto supportsModel = [](IParser& self, const py::buffer& model, const char* path = nullptr) {
+static const auto supportsModel = [](IParser& self, py::buffer const& model, char const* path = nullptr) {
    py::buffer_info info = model.request();
    SubGraphCollection_t subgraphs;
-    const bool supported = self.supportsModel(info.ptr, info.size * info.itemsize, subgraphs, path);
+    bool const supported = self.supportsModel(info.ptr, info.size * info.itemsize, subgraphs, path);
    return std::make_pair(supported, subgraphs);
 };

+static const auto supportsModelV2 = [](IParser& self, py::buffer const& model, char const* path = nullptr) {
+    py::buffer_info info = model.request();
+    return self.supportsModelV2(info.ptr, info.size * info.itemsize, path);
+};
+
+static const auto isSubgraphSupported
+    = [](IParser& self, int64_t const index) { return self.isSubgraphSupported(index); };
+
+static const auto getSubgraphNodes = [](IParser& self, int64_t const index) {
+    py::list py_nodes;
+    int64_t nb_nodes = 0;
+    int64_t* nodes = self.getSubgraphNodes(index, nb_nodes);
+    for (int64_t i = 0; i < nb_nodes; i++)
+    {
+        py_nodes.append(nodes[i]);
+    }
+    return py_nodes;
+};
+
 static const auto get_used_vc_plugin_libraries = [](IParser& self) {
    std::vector<std::string> vcPluginLibs;
    int64_t nbPluginLibs;
@@ -117,13 +136,13 @@ static const auto get_local_function_stack = [](IParserError& self) {
    return localFunctionStack;
 };

-static const auto refitFromBytes = [](IParserRefitter& self, const py::buffer& model, const char* path = nullptr) {
+static const auto refitFromBytes = [](IParserRefitter& self, py::buffer const& model, char const* path = nullptr) {
    py::buffer_info info = model.request();
    return self.refitFromBytes(info.ptr, info.size * info.itemsize, path);
 };

 static const auto refitFromFile
-    = [](IParserRefitter& self, const std::string& model) { return self.refitFromFile(model.c_str()); };
+    = [](IParserRefitter& self, std::string const& model) { return self.refitFromFile(model.c_str()); };

 } // namespace lambdas

@@ -143,6 +162,11 @@ void bindOnnx(py::module& m)
            py::call_guard<py::gil_scoped_release>{})
        .def("supports_operator", &IParser::supportsOperator, "op_name"_a, OnnxParserDoc::supports_operator)
        .def("supports_model", lambdas::supportsModel, "model"_a, "path"_a = nullptr, OnnxParserDoc::supports_model)
+        .def("supports_model_v2", lambdas::supportsModelV2, "model"_a, "path"_a = nullptr,
+            OnnxParserDoc::supports_model_v2)
+        .def_property_readonly("num_subgraphs", &IParser::getNbSubgraphs)
+        .def("is_subgraph_supported", lambdas::isSubgraphSupported, "index"_a, OnnxParserDoc::is_subgraph_supported)
+        .def("get_subgraph_nodes", lambdas::getSubgraphNodes, "index"_a, OnnxParserDoc::get_subgraph_nodes)
        .def_property_readonly("num_errors", &IParser::getNbErrors)
        .def("get_error", &IParser::getError, "index"_a, OnnxParserDoc::get_error)
        .def("clear_errors", &IParser::clearErrors, OnnxParserDoc::clear_errors)
@@ -44,7 +44,8 @@ size_t size(nvinfer1::DataType type)
    case nvinfer1::DataType::kUINT8: return 1;
    case nvinfer1::DataType::kFP8: return 1;
    case nvinfer1::DataType::kBF16: return 2;
-    case nvinfer1::DataType::kINT4: break; // TRT-22011 - need to address sub-byte element size
+    case nvinfer1::DataType::kINT4:
+        break; // TRT-22011 - need to address sub-byte element size
    }
    return -1;
 }
@@ -62,9 +63,10 @@ std::unique_ptr<py::dtype> nptype(nvinfer1::DataType type)
    case nvinfer1::DataType::kINT64: return makeDtype("i8");
    case nvinfer1::DataType::kBOOL: return makeDtype("b1");
    case nvinfer1::DataType::kUINT8: return makeDtype("u1");
-    case nvinfer1::DataType::kFP8: return nullptr;
-    case nvinfer1::DataType::kBF16: return nullptr;
-    case nvinfer1::DataType::kINT4: return nullptr;
+    case nvinfer1::DataType::kFP8:
+    case nvinfer1::DataType::kBF16:
+    case nvinfer1::DataType::kINT4:
+        return nullptr;
    }
    return nullptr;
 }