ENH: Performance monitoring for explainers (#5071)

* Init asv

Co-authored-by: Zhihao Dai <zhihao.dai@eng.ox.ac.uk>

* Add exact explainer suite

Co-authored-by: Zhihao Dai <zhihao.dai@eng.ox.ac.uk>

* Add partition explainer suite

Co-authored-by: Zhihao Dai <zhihao.dai@eng.ox.ac.uk>

* Add permutation explainer suite

Co-authored-by: Zhihao Dai <zhihao.dai@eng.ox.ac.uk>

* Clear out init py

Co-authored-by: Zhihao Dai <zhihao.dai@eng.ox.ac.uk>

* Remove basic suite

Co-authored-by: Zhihao Dai <zhihao.dai@eng.ox.ac.uk>

---------

Co-authored-by: Tobias Pitters <31857876+CloseChoice@users.noreply.github.com>

.gitignore

@@ -47,3 +47,7 @@ uv.lock
 
 # Auto-generated by nanobind_add_stub
 *.pyi
+
+# Performance monitoring using asv
+monitoring/html/
+monitoring/results/

asv.conf.json

@@ -0,0 +1,21 @@
+{
+    "version": 1,
+    "project": "shap",
+    "project_url": "https://github.com/shap/shap",
+    "repo": ".",
+    "branches": ["master"],
+    "environment_type": "uv",
+    "build_command": [
+        "python -m build --wheel --outdir {build_cache_dir} {build_dir}"
+    ],
+    "matrix": {
+        "req": {
+            "xgboost": []
+        }
+    },
+    "show_commit_url": "https://github.com/shap/shap/commit/",
+    "pythons": ["3.13"],
+    "benchmark_dir": "monitoring",
+    "html_dir": "monitoring/html",
+    "results_dir": "monitoring/results"
+}

monitoring/exact_explainer.py

@@ -0,0 +1,43 @@
+from xgboost import XGBClassifier
+
+from shap.datasets import adult
+from shap.explainers import ExactExplainer
+from shap.maskers import Partition
+
+
+class ExactSuite:
+    # Adapted from tests/explainers/test_exact.py
+    max_samples = 100
+
+    def setup(self):
+        self.model = XGBClassifier(tree_method="exact", base_score=0.5)
+
+        # get a dataset on income prediction
+        self.X, self.y = adult()
+        if self.max_samples is not None:
+            self.X = self.X.iloc[: self.max_samples]
+            self.y = self.y[: self.max_samples]
+        self.X = self.X.values
+
+        # fit the model on the data
+        self.model.fit(self.X, self.y)
+
+    def time_single_output(self):
+        ex = ExactExplainer(self.model.predict, self.X)
+        _ = ex(self.X)
+
+    def time_multi_output(self):
+        ex = ExactExplainer(self.model.predict_proba, self.X)
+        _ = ex(self.X)
+
+    def time_interactions(self):
+        ex = ExactExplainer(self.model.predict, self.X)
+        _ = ex(self.X, interactions=True)
+
+    def time_single_output_partition_masker(self):
+        ex = ExactExplainer(self.model.predict, masker=Partition(self.X))
+        _ = ex(self.X)
+
+    def time_multi_output_partition_masker(self):
+        ex = ExactExplainer(self.model.predict_proba, masker=Partition(self.X))
+        _ = ex(self.X)

monitoring/partition_explainer.py

@@ -0,0 +1,32 @@
+from xgboost import XGBClassifier
+
+from shap.datasets import adult
+from shap.explainers import PartitionExplainer
+
+
+class PartitionSuite:
+    # Adapted from tests/explainers/test_partition.py
+    # TODO: should we add translation tests here too?
+    # This would introduce a dependency on torch and transformers.
+    max_samples = 100
+
+    def setup(self):
+        self.model = XGBClassifier(tree_method="exact", base_score=0.5)
+
+        # get a dataset on income prediction
+        self.X, self.y = adult()
+        if self.max_samples is not None:
+            self.X = self.X.iloc[: self.max_samples]
+            self.y = self.y[: self.max_samples]
+        self.X = self.X.values
+
+        # fit the model on the data
+        self.model.fit(self.X, self.y)
+
+    def time_single_output(self):
+        ex = PartitionExplainer(self.model.predict, self.X)
+        _ = ex(self.X)
+
+    def time_multi_output(self):
+        ex = PartitionExplainer(self.model.predict_proba, self.X)
+        _ = ex(self.X)

monitoring/permutation_explainer.py

@@ -0,0 +1,39 @@
+from xgboost import XGBClassifier
+
+from shap.datasets import adult
+from shap.explainers import PermutationExplainer
+from shap.maskers import Partition
+
+
+class PermutationSuite:
+    # Adapted from tests/explainers/test_permutation.py
+    max_samples = 100
+
+    def setup(self):
+        self.model = XGBClassifier(tree_method="exact", base_score=0.5)
+
+        # get a dataset on income prediction
+        self.X, self.y = adult()
+        if self.max_samples is not None:
+            self.X = self.X.iloc[: self.max_samples]
+            self.y = self.y[: self.max_samples]
+        self.X = self.X.values
+
+        # fit the model on the data
+        self.model.fit(self.X, self.y)
+
+    def time_single_output(self):
+        ex = PermutationExplainer(self.model.predict, self.X)
+        _ = ex(self.X)
+
+    def time_multi_output(self):
+        ex = PermutationExplainer(self.model.predict_proba, self.X)
+        _ = ex(self.X)
+
+    def time_single_output_partition_masker(self):
+        ex = PermutationExplainer(self.model.predict, masker=Partition(self.X))
+        _ = ex(self.X)
+
+    def time_multi_output_partition_masker(self):
+        ex = PermutationExplainer(self.model.predict_proba, masker=Partition(self.X))
+        _ = ex(self.X)

pyproject.toml

@@ -102,6 +102,8 @@ test = [
   # that were removed in scikit-learn 1.9. See https://github.com/uber/causalml/issues/926
   "scikit-learn<1.9",
   "selenium",  # needed to test the javascript based plots
+  # performance monitoring for time-critical explainers
+  "asv",
 ]
 nbtest = [
   "jupyter",