Merge branch 'main' into qnn_2_30

csharp/tools/MauiModelTester/create_test_data.py

@@ -2,7 +2,6 @@
 import shutil
 import sys
 from pathlib import Path
-from typing import Dict, List, Optional
 
 import numpy as np
 
@@ -84,7 +83,7 @@ def parse_args():
     return args
 
 
-def create_existing_data_map(pb_files: List[Path]):
+def create_existing_data_map(pb_files: list[Path]):
     import onnx_test_data_utils as data_utils
 
     data_map = {}
@@ -98,9 +97,9 @@ def create_existing_data_map(pb_files: List[Path]):
 
 def add_model_and_test_data_to_app(
     model_path: Path,
-    symbolic_dims: Optional[Dict[str, int]] = None,
-    input_map: Optional[Dict[str, np.ndarray]] = None,
-    output_map: Optional[Dict[str, np.ndarray]] = None,
+    symbolic_dims: dict[str, int] | None = None,
+    input_map: dict[str, np.ndarray] | None = None,
+    output_map: dict[str, np.ndarray] | None = None,
 ):
     import ort_test_dir_utils as utils
 

docs/python/_common/onnx_sphinx.py

@@ -282,7 +282,7 @@ def get_domain_list():
     """
     Returns the list of available domains.
     """
-    return list(sorted(set(map(lambda s: s.domain, get_all_schemas_with_history()))))
+    return sorted({s.domain for s in get_all_schemas_with_history()})
 
 
 def get_operator_schemas(op_name, version=None, domain=None):
@@ -779,9 +779,9 @@ def render(self, indent=""):
                 name = op["name"]
                 dom = self.domain.replace(".", "-")
                 table_dom.append(f"    * - :ref:`l-onnx-doc{dom}-{name}`")
-                versions = list(reversed(sorted((k, v) for k, v in op["links"].items() if isinstance(k, int))))
+                versions = sorted(((k, v) for k, v in op["links"].items() if isinstance(k, int)), reverse=True)
                 col1 = ", ".join(f":ref:`{k} <{v}>`" for k, v in versions)
-                diffs = list(reversed(sorted((k, v) for k, v in op["links"].items() if isinstance(k, tuple))))
+                diffs = sorted(((k, v) for k, v in op["links"].items() if isinstance(k, tuple)), reverse=True)
                 col2 = ", ".join(f":ref:`{k[1]}/{k[0]} <{v}>`" for k, v in diffs)
                 table_dom.append(f"      - {col1}")
                 table_dom.append(f"      - {col2}")

onnxruntime/core/optimizer/identical_children_consolidation.cc

@@ -45,7 +45,7 @@ std::vector<std::vector<NodeIndex>> IdenticalChildrenConsolidation::DivideIdenti
     const Graph& graph,
     Node* node,
     const string_view& op) {
-  unordered_map<string_view, std::vector<NodeIndex>> identical_children_map;
+  unordered_map<std::string, std::vector<NodeIndex>> identical_children_map;
   for (auto i = node->OutputEdgesBegin(); i != node->OutputEdgesEnd(); ++i) {
     if (i->GetNode().OpType() == op) {
       identical_children_map[IdentityBuilder(graph, i->GetNode())].push_back(i->GetNode().Index());
@@ -125,4 +125,4 @@ std::string IdenticalChildrenConsolidation::IdentityBuilder(const Graph& graph,
 
   return identity.str();
 }
-}  // namespace onnxruntime
+}  // namespace onnxruntime

onnxruntime/core/providers/cpu/nn/batch_norm.h

@@ -75,9 +75,11 @@ class BatchNorm : public OpKernel {
     const TensorShape& x_shape = X->Shape();
     Tensor* Y = p_op_kernel_context->Output(0, x_shape);
 
+    // X shape is [N, C, D1, D2, ... Dn], but it can also be 1-D according to onnx spec:
+    // "The op also accepts single dimension input of size N in which case C is assumed to be 1"
     const auto& dims_vec = x_shape.GetDims();
     const size_t N = onnxruntime::narrow<size_t>(dims_vec[0]);
-    const size_t C = onnxruntime::narrow<size_t>(dims_vec[1]);  // assume NCHW as per the spec
+    const size_t C = dims_vec.size() == 1 ? 1 : onnxruntime::narrow<size_t>(dims_vec[1]);
 
     // calculate sample_size (per individual channel)
     size_t sample_size = 1;

onnxruntime/core/providers/cpu/nn/batch_norm_helper.h

@@ -28,6 +28,9 @@ class BatchNormHelper {
     // NHWC dependent shape: X
     // All other shapes are assumed to be in NCHW layout?
     const auto& x_dims = X->Shape().GetDims();
+    if (x_dims.size() < 1) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input X: NumDimensions() < 1");
+    }
 
     // If x_dims size < 2, num_channels defaults to 1.
     int64_t num_channels;

onnxruntime/core/providers/qnn/builder/opbuilder/simple_op_builder.cc

@@ -22,11 +22,6 @@ class SimpleOpBuilder : public BaseOpBuilder {
   ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SimpleOpBuilder);
 
  protected:
-  Status ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
-                       const NodeUnit& node_unit,
-                       const logging::Logger& logger,
-                       std::vector<std::string>& input_names,
-                       bool do_op_validation) const override ORT_MUST_USE_RESULT;
   Status ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wrapper,
                                      const NodeUnit& node_unit,
                                      std::vector<std::string>&& input_names,
@@ -53,91 +48,6 @@ class SimpleOpBuilder : public BaseOpBuilder {
   static constexpr std::array<std::string_view, 3> gridsample_supported_padding_modes = {"zeros", "border", "reflection"};
 };
 
-// Move to qnn_utils if it's re-usable
-Status InsertConvertOp(QnnModelWrapper& qnn_model_wrapper,
-                       const std::string& convert_input_name,
-                       const std::string& convert_output_name,
-                       Qnn_DataType_t input_qnn_data_type,
-                       Qnn_DataType_t output_qnn_data_type,
-                       int32_t input_offset,
-                       float input_scale,
-                       const std::vector<uint32_t>& output_shape,
-                       bool do_op_validation) {
-  // Assume input is already handled.
-  float qmin = 0.0f;
-  float qmax = 255.0f;
-  ORT_RETURN_IF_ERROR(qnn::utils::GetQminQmax(input_qnn_data_type, qmin, qmax));
-  double value_min = qnn::utils::Dequantize(input_offset, input_scale, qmin);
-  double value_max = qnn::utils::Dequantize(input_offset, input_scale, qmax);
-  float scale = 0.0f;
-  int32_t offset = 0;
-  ORT_RETURN_IF_ERROR(qnn::utils::GetQuantParams(static_cast<float>(value_min),
-                                                 static_cast<float>(value_max),
-                                                 output_qnn_data_type,
-                                                 scale,
-                                                 offset));
-
-  std::vector<uint32_t> output_shape_copy = output_shape;
-  QnnTensorWrapper convert_output_tensorwrapper(convert_output_name,
-                                                QNN_TENSOR_TYPE_NATIVE,
-                                                output_qnn_data_type,
-                                                QnnQuantParamsWrapper(scale, offset),
-                                                std::move(output_shape_copy));
-  ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(convert_output_tensorwrapper)), "Failed to add tensor.");
-
-  ORT_RETURN_IF_NOT(qnn_model_wrapper.CreateQnnNode(convert_output_name,
-                                                    QNN_OP_PACKAGE_NAME_QTI_AISW,
-                                                    "Convert",
-                                                    {convert_input_name},
-                                                    {convert_output_name},
-                                                    {},
-                                                    do_op_validation),
-                    "Failed to add node.");
-  return Status::OK();
-}
-
-Status SimpleOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
-                                      const NodeUnit& node_unit,
-                                      const logging::Logger& logger,
-                                      std::vector<std::string>& input_names,
-                                      bool do_op_validation) const {
-  const std::string& op_type = node_unit.OpType();
-  ORT_RETURN_IF_ERROR(BaseOpBuilder::ProcessInputs(qnn_model_wrapper, node_unit, logger, input_names, do_op_validation));
-
-  if (op_type == "MatMul") {
-    const auto& inputs = node_unit.Inputs();
-    TensorInfo input0_info = {};
-    TensorInfo input1_info = {};
-    ORT_RETURN_IF_ERROR(qnn_model_wrapper.GetTensorInfo(inputs[0], input0_info));
-    ORT_RETURN_IF_ERROR(qnn_model_wrapper.GetTensorInfo(inputs[1], input1_info));
-    // Need to insert Convert op if both inputs are dynamic inputs and are ufixed_16
-    if (!input0_info.is_initializer && !input1_info.is_initializer &&
-        input0_info.qnn_data_type == input1_info.qnn_data_type &&
-        input0_info.qnn_data_type == QNN_DATATYPE_UFIXED_POINT_16) {
-      ORT_RETURN_IF_NOT(input1_info.quant_param.IsPerTensor(),
-                        "MatMul's activation inputs only support per-tensor quantization");
-      const Qnn_QuantizeParams_t& quant_param = input1_info.quant_param.Get();
-      // insert Convert op after input1
-      std::string convert_input_name = input_names.back();
-      input_names.pop_back();
-      const std::string& matmul_output_name = node_unit.Outputs()[0].node_arg.Name();
-      std::string convert_output_name = convert_input_name + "_convert_" + matmul_output_name;
-      ORT_RETURN_IF_ERROR(InsertConvertOp(qnn_model_wrapper,
-                                          convert_input_name,
-                                          convert_output_name,
-                                          input1_info.qnn_data_type,
-                                          QNN_DATATYPE_UFIXED_POINT_8,
-                                          quant_param.scaleOffsetEncoding.offset,
-                                          quant_param.scaleOffsetEncoding.scale,
-                                          input1_info.shape,
-                                          do_op_validation));
-      input_names.push_back(convert_output_name);
-    }
-  }
-
-  return Status::OK();
-}
-
 Status SimpleOpBuilder::ExplicitOpCheck(QnnModelWrapper& qnn_model_wrapper,
                                         const NodeUnit& node_unit) const {
   const std::string& op_type = node_unit.OpType();
@@ -378,19 +288,6 @@ Status SimpleOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_w
     ORT_RETURN_IF(norm_p_order != 2, "QNN EP only supports LpNormalization with 'p' attribute equal to 2.");
   }
 
-  if (op_type == "MatMul") {
-    Qnn_Scalar_t scalar_param = QNN_SCALAR_INIT;
-    scalar_param.dataType = QNN_DATATYPE_BOOL_8;
-    scalar_param.bool8Value = 0;
-    QnnParamWrapper transpose_in0_param(node_unit.Index(), node_unit.Name(), QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN0, scalar_param);
-    param_tensor_names.push_back(transpose_in0_param.GetParamTensorName());
-    qnn_model_wrapper.AddParamWrapper(std::move(transpose_in0_param));
-
-    QnnParamWrapper transpose_in1_param(node_unit.Index(), node_unit.Name(), QNN_OP_MAT_MUL_PARAM_TRANSPOSE_IN1, scalar_param);
-    param_tensor_names.push_back(transpose_in1_param.GetParamTensorName());
-    qnn_model_wrapper.AddParamWrapper(std::move(transpose_in1_param));
-  }
-
   if (op_type == "LeakyRelu") {
     std::string input_name = "alpha";
     ORT_RETURN_IF_ERROR(ProcessAlphaAttributeAsInput(qnn_model_wrapper, node_unit, input_name));

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc

@@ -1099,35 +1099,39 @@ Status QnnBackendManager::TerminateQnnLog() {
 }
 
 void QnnBackendManager::ReleaseResources() {
+  if (!backend_setup_completed_) {
+    return;
+  }
+
   auto result = ReleaseContext();
   if (Status::OK() != result) {
-    LOGS_DEFAULT(ERROR) << "Failed to ReleaseContext: " << result.ErrorMessage();
+    LOGS_DEFAULT(ERROR) << "Failed to ReleaseContext.";
   }
 
   result = ReleaseProfilehandle();
   if (Status::OK() != result) {
-    LOGS_DEFAULT(ERROR) << "Failed to ReleaseProfilehandle: " << result.ErrorMessage();
+    LOGS_DEFAULT(ERROR) << "Failed to ReleaseProfilehandle.";
   }
 
   result = ReleaseDevice();
   if (Status::OK() != result) {
-    LOGS_DEFAULT(ERROR) << "Failed to ReleaseDevice: " << result.ErrorMessage();
+    LOGS_DEFAULT(ERROR) << "Failed to ReleaseDevice.";
   }
 
   result = ShutdownBackend();
   if (Status::OK() != result) {
-    LOGS_DEFAULT(ERROR) << "Failed to ShutdownBackend: " << result.ErrorMessage();
+    LOGS_DEFAULT(ERROR) << "Failed to ShutdownBackend.";
   }
 
   result = TerminateQnnLog();
   if (Status::OK() != result) {
-    LOGS_DEFAULT(ERROR) << "Failed to TerminateQnnLog: " << result.ErrorMessage();
+    LOGS_DEFAULT(ERROR) << "Failed to TerminateQnnLog.";
   }
 
   if (backend_lib_handle_) {
     result = UnloadLib(backend_lib_handle_);
     if (Status::OK() != result) {
-      LOGS_DEFAULT(ERROR) << "Failed to unload backend library: " << result.ErrorMessage();
+      LOGS_DEFAULT(ERROR) << "Failed to unload backend library.";
     }
   }
 

onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.cc

@@ -75,6 +75,20 @@ Status QnnModelWrapper::MakeTensorWrapper(const NodeUnitIODef& tensor, QnnTensor
   return Status::OK();
 }
 
+Status QnnModelWrapper::MakeTensorWrapper(const TensorInfo& tensor_info,
+                                          const std::string& tensor_name,
+                                          QnnTensorWrapper& tensor_wrapper) const {
+  std::vector<uint8_t> unpacked_tensor;
+  if (tensor_info.is_initializer) {
+    ORT_RETURN_IF_ERROR(UnpackInitializerData(*tensor_info.initializer_tensor, unpacked_tensor));
+  }
+
+  tensor_wrapper = QnnTensorWrapper(tensor_name, GetTensorType(tensor_name), tensor_info.qnn_data_type,
+                                    tensor_info.quant_param.Copy(), std::vector<uint32_t>(tensor_info.shape),
+                                    std::move(unpacked_tensor));
+  return Status::OK();
+}
+
 bool QnnModelWrapper::AddTensorWrapper(QnnTensorWrapper&& tensor_wrapper) {
   // Keep a copy of tensor name sine it will be moved with the wrapper into model_tensors_map_
   std::string tensor_name = tensor_wrapper.GetName();

onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h

@@ -66,6 +66,9 @@ class QnnModelWrapper {
 
   // Make a QnnTensorWrapper from an onnx input or output.
   Status MakeTensorWrapper(const NodeUnitIODef& tensor, QnnTensorWrapper& tensor_wrapper) const;
+  Status MakeTensorWrapper(const TensorInfo& tensor_info,
+                           const std::string& tensor_name,
+                           QnnTensorWrapper& tensor_wrapper) const;
 
   // Add to internal tensor wrapper table
   bool AddTensorWrapper(QnnTensorWrapper&& tensor_wrapper);

onnxruntime/python/backend/backend_rep.py

@@ -6,8 +6,6 @@
 Implements ONNX's backend API.
 """
 
-from typing import Any, Tuple  # noqa: F401
-
 from onnx.backend.base import BackendRep
 
 from onnxruntime import RunOptions

onnxruntime/python/onnxruntime_inference_collection.py

@@ -9,7 +9,8 @@
 import os
 import typing
 import warnings
-from typing import Any, Sequence
+from collections.abc import Sequence
+from typing import Any
 
 from onnxruntime.capi import _pybind_state as C
 
@@ -137,19 +138,19 @@ def set_provider_options(name, options):
         if len(providers) != len(provider_options):
             raise ValueError("'providers' and 'provider_options' should be the same length if both are given.")
 
-        if not all([isinstance(provider, str) for provider in providers]):
+        if not all(isinstance(provider, str) for provider in providers):
             raise ValueError("Only string values for 'providers' are supported if 'provider_options' is given.")
 
-        if not all([isinstance(options_for_provider, dict) for options_for_provider in provider_options]):
+        if not all(isinstance(options_for_provider, dict) for options_for_provider in provider_options):
             raise ValueError("'provider_options' values must be dicts.")
 
-        for name, options in zip(providers, provider_options):
+        for name, options in zip(providers, provider_options, strict=False):
             set_provider_options(name, options)
 
     else:
         for provider in providers:
             if isinstance(provider, str):
-                set_provider_options(provider, dict())
+                set_provider_options(provider, {})
             elif (
                 isinstance(provider, tuple)
                 and len(provider) == 2

onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py

@@ -22,7 +22,6 @@
 import os
 import sys
 from dataclasses import dataclass
-from typing import List, Optional, Union
 
 import onnx
 from onnx import TensorProto, helper
@@ -65,7 +64,7 @@ class IOInfo:
     index: int
     name: str
     elem_type: TensorProto.DataType
-    shape: Optional[List[Union[int, str]]]
+    shape: list[int | str] | None
 
 
 def str_is_int(string: str) -> bool:
@@ -76,7 +75,7 @@ def str_is_int(string: str) -> bool:
         return False
 
 
-def parse_shape(shape_str: str) -> Optional[List[Union[int, str]]]:
+def parse_shape(shape_str: str) -> list[int | str] | None:
     try:
         shape = [int(s) if str_is_int(s) else s for s in shape_str.split(",")]
     except ValueError:
@@ -204,7 +203,7 @@ def parse_arguments() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def get_attributes(attr_data_info: List[List[str]]):
+def get_attributes(attr_data_info: list[list[str]]):
     if not attr_data_info:
         return {}
 

onnxruntime/python/tools/kernel_explorer/kernels/kernel_explorer.py

@@ -13,11 +13,11 @@
 import sys
 from abc import abstractmethod
 from argparse import Action, ArgumentParser
+from collections.abc import Callable
 from contextlib import contextmanager
 from dataclasses import dataclass
 from fnmatch import fnmatch
 from functools import wraps
-from typing import Callable
 
 build_dir = os.environ.get("KERNEL_EXPLORER_BUILD_DIR", None)
 if build_dir is None:
@@ -220,7 +220,7 @@ def set_dispatch(name):
         from difflib import SequenceMatcher as Matcher
 
         valid_names = list(_ke_context.dispatchable.keys())
-        scored_names = list(reversed(sorted([(Matcher(None, name, a).ratio(), a) for a in valid_names])))
+        scored_names = sorted([(Matcher(None, name, a).ratio(), a) for a in valid_names], reverse=True)
         top10 = "\n    ".join([a for _, a in scored_names[:10]])
         msg = f"'{name}' is not registered for dispatch. Top 10 matches are:\n    {top10}"
         print(msg)