github-featured/roboflow--rf-detr
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

perf(evaluation): share bbox IoU per image (#1373)

Browse Source 
* perf(evaluation): share bbox IoU per image
* fix(evaluation): unify tie-break contract and preserve score precision
* refactor(evaluation): collapse dead bbox branch and hoist iou_type dispatch
* test(evaluation): add bbox differential oracle and matcher unit tests
* perf(evaluation): add C=1 and no-crowd fast paths, reduce per-class dispatch
* style(evaluation): align empty-GT sort to kind="stable"

---------

Co-authored-by: Codex <codex@openai.com>
Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
This commit is contained in:
Jirka Borovec
2026-08-20 10:32:55 +02:00
committed by GitHub
parent 00639715a7
commit ae900cd96a
2 changed files with 980 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rfdetr/evaluation/matching.py
+392 -90
View File
@@ -18,7 +18,7 @@
from __future__ import annotations
from collections import Counter
from typing import Any, cast
from typing import Any, TypeAlias, cast
import numpy as np
import torch
@@ -28,6 +28,27 @@ from torchvision.ops import box_iou
from rfdetr.utilities import all_gather
#: One class's contribution to the accumulator: its detection scores in descending order, their
#: true-positive flags, their crowd-ignore flags, and the class's non-crowd GT count.
_ClassContribution: TypeAlias = tuple[
np.ndarray[Any, np.dtype[np.float32]],
np.ndarray[Any, np.dtype[np.int64]],
np.ndarray[Any, np.dtype[np.bool_]],
int,
]
#: Positions of one class inside an image's prediction and GT arrays: the rows and the columns it
#: owns in the image-wide ``box_iou`` matrix. ``None`` instead means the image holds a single class,
#: which owns the whole matrix and needs no positions at all.
_ClassSlice: TypeAlias = tuple[
np.ndarray[Any, np.dtype[np.intp]],
np.ndarray[Any, np.dtype[np.intp]],
]
#: Column positions of a class that has predictions but no GT of its own. Shared, so the per-class
#: lookup that misses can hand back an empty selection without allocating one every time.
_NO_INDICES: np.ndarray[Any, np.dtype[np.intp]] = np.zeros(0, dtype=np.intp)
def _compute_mask_iou(pred_masks: Tensor, gt_masks: Tensor) -> Tensor:
"""Compute pairwise boolean-mask IoU between N predictions and M ground truths.
@@ -53,31 +74,223 @@ def _compute_mask_iou(pred_masks: Tensor, gt_masks: Tensor) -> Tensor:
return torch.where(union > 0, inter / union, torch.zeros_like(inter))
def _match_single_class(
pred_scores: Tensor,
pred_items: Tensor,
gt_items: Tensor,
gt_crowd: Tensor,
def _match_sorted_iou_matrix(
iou_matrix_sorted: np.ndarray[Any, np.dtype[np.float32]],
gt_crowd_np: np.ndarray[Any, np.dtype[np.bool_]],
iou_threshold: float,
iou_type: str,
) -> tuple[
np.ndarray[Any, np.dtype[np.float32]],
np.ndarray[Any, np.dtype[np.int64]],
np.ndarray[Any, np.dtype[np.bool_]],
int,
]:
"""Greedy highest-score-first matching for one class in one image.
"""Apply COCO greedy matching to score-ordered NumPy IoUs for one class.
Implements the COCO matching algorithm: each GT is matched at most once; detections are processed in descending
score order; detections matched to crowd GTs are marked as ignored rather than false positives.
Dispatches on whether the class has any crowd GT. Both loops below are per-detection Python
loops whose cost is dominated by the number of NumPy calls each iteration makes, so the common
crowd-free case gets its own loop rather than paying for masks that can only be no-ops.
Args:
iou_matrix_sorted: Pairwise IoUs whose rows are in descending detection-score order. Must not
be modified here: the bbox caller derives it from the one ``box_iou`` matrix shared by
every class of an image, so masking rows in place instead of copying them would leak one
class's matching state into the classes matched after it.
gt_crowd_np: Boolean crowd mask aligned to IoU columns.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Tuple of true-positive flags, crowd-ignore flags, and the number of non-crowd ground truths.
The scores the rows were ordered by are not returned — every caller already holds them.
"""
if not gt_crowd_np.any():
return _match_rows_without_crowd(iou_matrix_sorted, iou_threshold)
return _match_rows_with_crowd(iou_matrix_sorted, gt_crowd_np, iou_threshold)
def _match_rows_without_crowd(
iou_matrix_sorted: np.ndarray[Any, np.dtype[np.float32]],
iou_threshold: float,
) -> tuple[
np.ndarray[Any, np.dtype[np.int64]],
np.ndarray[Any, np.dtype[np.bool_]],
int,
]:
"""Greedily match score-ordered IoUs for a class whose ground truths are all non-crowd.
Behaviourally a special case of ``_match_rows_with_crowd()`` — every GT is claimable, none can
be ignored, and ``total_gt`` is simply the column count. What makes it worth its own loop is
that the per-detection cost here is NumPy call overhead, not array work: at a few dozen GTs a
call costs more than the elements it touches, so the win comes from making calls disappear.
Two whole-matrix reductions do that for the two cases that need no search at all: a detection
whose best IoU is below the threshold cannot match any GT, and one whose best GT is still free
matches exactly that GT (masking can only lower other columns, so it cannot promote a different
one, and ``argmax`` already resolved ties to the lowest column index). Only a detection whose
best GT was claimed by a higher-scoring detection falls back to a masked search.
Args:
iou_matrix_sorted: Pairwise IoUs whose rows are in descending detection-score order. Read
only, for the reason given on ``_match_sorted_iou_matrix()``.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Tuple of true-positive flags, all-False crowd-ignore flags, and the ground-truth count.
"""
n, m = iou_matrix_sorted.shape
gt_matched_np = np.zeros(m, dtype=np.bool_)
pred_match_np = np.zeros(n, dtype=np.int64)
# argmax before max, so that a matrix with no GT columns still fails with the same "argmax of
# an empty sequence" ValueError the per-detection search raises.
best_columns = cast(list[int], iou_matrix_sorted.argmax(axis=1).tolist())
best_ious = cast(list[float], iou_matrix_sorted.max(axis=1).tolist())
for index in range(n):
if best_ious[index] < iou_threshold:
continue
best_column = best_columns[index]
if not gt_matched_np[best_column]:
pred_match_np[index] = 1
gt_matched_np[best_column] = True
continue
# Copy-then-mask rather than np.where(): measured cheaper, and it leaves the source matrix
# — shared across the classes of an image on the bbox path — untouched either way.
free_ious = cast(np.ndarray[Any, np.dtype[np.float32]], iou_matrix_sorted[index].copy())
free_ious[gt_matched_np] = -1.0
best_index = int(free_ious.argmax())
if free_ious[best_index] >= iou_threshold:
pred_match_np[index] = 1
gt_matched_np[best_index] = True
return pred_match_np, np.zeros(n, dtype=np.bool_), m
def _match_rows_with_crowd(
iou_matrix_sorted: np.ndarray[Any, np.dtype[np.float32]],
gt_crowd_np: np.ndarray[Any, np.dtype[np.bool_]],
iou_threshold: float,
) -> tuple[
np.ndarray[Any, np.dtype[np.int64]],
np.ndarray[Any, np.dtype[np.bool_]],
int,
]:
"""Greedily match score-ordered IoUs for a class that may have crowd ground truths.
Handles crowd GTs in full generality, so it stays correct for an all-non-crowd input too — that
is what makes it the reference ``_match_rows_without_crowd()`` is differentially tested against.
Args:
iou_matrix_sorted: Pairwise IoUs whose rows are in descending detection-score order. Read
only, for the reason given on ``_match_sorted_iou_matrix()``.
gt_crowd_np: Boolean crowd mask aligned to IoU columns.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Tuple of true-positive flags, crowd-ignore flags, and the number of non-crowd ground truths.
"""
n, m = iou_matrix_sorted.shape
gt_matched_np = np.zeros(m, dtype=np.bool_)
pred_match_np = np.zeros(n, dtype=np.int64)
pred_ignore_np = np.zeros(n, dtype=np.bool_)
any_crowd = bool(gt_crowd_np.any())
not_crowd_np = ~gt_crowd_np
# Each detection can claim at most one non-crowd target, so score order
# remains sequential even though the IoU matrix is already vectorized.
for index in range(len(iou_matrix_sorted)):
ious = cast(np.ndarray[Any, np.dtype[np.float32]], iou_matrix_sorted[index])
noncrowd_ious = ious.copy()
noncrowd_ious[gt_crowd_np] = -1.0
noncrowd_ious[gt_matched_np & not_crowd_np] = -1.0
# The bound method, not np.argmax(): the module-level function re-dispatches on its
# argument, which costs more per call than the search over a few dozen GTs.
best_noncrowd_index = int(noncrowd_ious.argmax())
if noncrowd_ious[best_noncrowd_index] >= iou_threshold:
pred_match_np[index] = 1
gt_matched_np[best_noncrowd_index] = True
elif any_crowd:
crowd_ious = ious.copy()
crowd_ious[not_crowd_np] = -1.0
if crowd_ious.max() >= iou_threshold:
pred_ignore_np[index] = True
return pred_match_np, pred_ignore_np, int(not_crowd_np.sum())
def _group_indices_by_label(
label_ids_np: np.ndarray[Any, np.dtype[np.int64]],
) -> dict[int, np.ndarray[Any, np.dtype[np.intp]]]:
"""Group array positions by label in one pass over an image's labels.
Replaces one ``label_ids_np == class_id`` scan per class — O(N) per class, O(N*C) over an
image — with a single stable sort. Within each group the positions stay ascending, which is the
order a per-class scan produces and the order the greedy matcher's stable score sort then
breaks ties on, so the grouping is a pure speedup and not a reordering.
Args:
label_ids_np: Label of every detection (or of every GT) of one image.
Returns:
Dict mapping each label present to its ascending positions in *label_ids_np*; empty for an
empty input.
"""
if label_ids_np.size == 0:
return {}
order = np.argsort(label_ids_np, kind="stable")
sorted_labels = label_ids_np[order]
boundaries = np.flatnonzero(sorted_labels[1:] != sorted_labels[:-1]) + 1
return {int(label_ids_np[group[0]]): group for group in np.split(order, boundaries)}
def _unmatched_contribution(
scores_np: np.ndarray[Any, np.dtype[np.floating[Any]]],
) -> _ClassContribution:
"""Build the contribution of a class that has detections but no ground truth.
Both ``iou_type`` paths reach this case and must report it identically: every detection is a
false positive, none is crowd-ignored, and the class adds nothing to the GT denominator.
Args:
scores_np: Detection scores of one class, in the caller's own float dtype so that near-tied
scores are ordered at full precision before the float32 cast.
Returns:
Tuple ``(scores_np, matches_np, ignore_np, total_gt)`` with the scores in descending order,
all-zero matches, all-False ignore flags, and a ``total_gt`` of 0.
"""
n = len(scores_np)
# kind="stable" costs nothing here — every match is 0 and every ignore is False regardless of
# tie order, since there is no GT to match against — but it keeps this the only unstable sort
# site in the file, instead of a silent third tie-break convention alongside the other two.
order = np.argsort(-scores_np, kind="stable")
return (
scores_np[order].astype(np.float32, copy=False),
np.zeros(n, dtype=np.int64),
np.zeros(n, dtype=np.bool_),
0,
)
def _match_single_class_segm(
pred_scores: Tensor,
pred_masks: Tensor,
gt_masks: Tensor,
gt_crowd: Tensor,
iou_threshold: float,
) -> _ClassContribution:
"""Greedy highest-score-first mask matching for one class in one image.
Implements the COCO matching algorithm over boolean-mask IoU: each GT is matched at most once; detections are
processed in descending score order; detections matched to crowd GTs are marked as ignored rather than false
positives. This is the segmentation path only — the box path never reaches here, because it shares one image-wide
``box_iou`` matrix and calls ``_match_sorted_iou_matrix`` directly.
Args:
pred_scores: Float tensor of shape [N] with detection confidences.
pred_items: Predictions — boxes [N, 4] in xyxy coords or masks [N, H, W].
gt_items: Ground truths — boxes [M, 4] in xyxy coords or masks [M, H, W].
pred_masks: Boolean prediction masks of shape [N, H, W].
gt_masks: Boolean ground-truth masks of shape [M, H, W].
gt_crowd: Bool tensor of shape [M], True for crowd instances.
iou_threshold: Minimum IoU to count as a positive match.
iou_type: ``"bbox"`` for box IoU or ``"segm"`` for mask IoU.
Returns:
Tuple ``(scores_np, matches_np, ignore_np, total_gt)`` where:
@@ -86,17 +299,15 @@ def _match_single_class(
- ignore_np: bool array [N], True if matched to a crowd GT.
- total_gt: number of non-crowd GT instances.
"""
n = pred_scores.shape[0]
m = gt_items.shape[0]
sort_idx = torch.argsort(pred_scores, descending=True)
# stable=True keeps tied scores in input order, which is the tie rule the bbox path gets from
# np.argsort(kind="stable") and PostProcess._select_topk gets from the same flag. torch's
# default sort is unstable past its ~32-element cutoff, so without it the greedy winner of a
# tie -- and with it the TP/FP split -- would depend on the sort backend rather than the input.
sort_idx = torch.argsort(pred_scores, descending=True, stable=True)
pred_scores_sorted = pred_scores[sort_idx]
pred_sorted = pred_items[sort_idx]
pred_sorted = pred_masks[sort_idx]
if iou_type == "bbox":
iou_matrix = box_iou(pred_sorted, gt_items) # [N, M]
else:
iou_matrix = _compute_mask_iou(pred_sorted, gt_items) # [N, M]
iou_matrix = _compute_mask_iou(pred_sorted, gt_masks) # [N, M]
# The greedy matching below is inherently sequential (each GT can only be claimed
# once, so iteration i depends on the outcome of i-1) — it cannot be vectorized away.
@@ -108,42 +319,113 @@ def _match_single_class(
iou_matrix_np = iou_matrix.detach().float().cpu().numpy() # [N, M] -- float() guards bf16/fp16 (no numpy dtype)
gt_crowd_np = gt_crowd.detach().cpu().numpy() # [M]
gt_matched_np = np.zeros(m, dtype=np.bool_)
pred_match_np = np.zeros(n, dtype=np.int64)
pred_ignore_np = np.zeros(n, dtype=np.bool_)
any_crowd = bool(gt_crowd_np.any())
not_crowd_np = ~gt_crowd_np # crowd mask is loop-invariant — compute the negation once
for i in range(n):
ious = iou_matrix_np[i] # [M]
# Try to match to a non-crowd GT (each non-crowd GT matched at most once).
nc_ious = ious.copy()
nc_ious[gt_crowd_np] = -1.0
nc_ious[gt_matched_np & not_crowd_np] = -1.0 # already claimed
best_nc_idx = int(np.argmax(nc_ious))
best_nc_iou = nc_ious[best_nc_idx]
if best_nc_iou >= iou_threshold:
pred_match_np[i] = 1
gt_matched_np[best_nc_idx] = True
# A detection matched to a crowd GT is ignored (not a false positive).
elif any_crowd:
crowd_ious = ious.copy()
crowd_ious[not_crowd_np] = -1.0
if crowd_ious.max() >= iou_threshold:
pred_ignore_np[i] = True
# else: false positive — pred_match_np stays 0
total_gt = int((~gt_crowd_np).sum())
matches_np, ignore_np, total_gt = _match_sorted_iou_matrix(iou_matrix_np, gt_crowd_np, iou_threshold)
return (
np.asarray(pred_scores_sorted.float().cpu().numpy(), dtype=np.float32),
pred_match_np,
pred_ignore_np,
np.asarray(pred_scores_sorted.detach().float().cpu().numpy(), dtype=np.float32),
matches_np,
ignore_np,
total_gt,
)
def _accumulate_bbox_class(
pred_scores_np: np.ndarray[Any, np.dtype[np.floating[Any]]],
class_slice: _ClassSlice | None,
gt_crowd_np: np.ndarray[Any, np.dtype[np.bool_]],
bbox_iou_matrix_np: np.ndarray[Any, np.dtype[np.float32]] | None,
iou_threshold: float,
) -> _ClassContribution:
"""Accumulate one class of one image against the image's shared box-IoU matrix.
Everything needed here is already on the host, so the class is selected by slicing the per-image
NumPy arrays instead of by indexing device tensors. A *class_slice* of ``None`` says the image
holds one class only: it then owns every row and column of *bbox_iou_matrix_np*, and the
two-axis fancy-index copy that isolates a class becomes a copy of the whole matrix onto itself,
so only the rows are reordered and the crowd mask is taken as it stands.
Args:
pred_scores_np: Every detection score of the image, in the caller's own float dtype.
class_slice: The class's prediction rows and GT columns, or ``None`` when the image holds a
single class. A class with predictions but no GT gets an empty column selection, never
``None``.
gt_crowd_np: Crowd flag per GT of the image.
bbox_iou_matrix_np: The image's ``box_iou`` matrix, or ``None`` when the image has no
detections or no GT at all. A class without GT returns before it is read.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Tuple ``(scores_np, matches_np, ignore_np, total_gt)`` contributed by the class.
"""
p_scores_np = pred_scores_np if class_slice is None else pred_scores_np[class_slice[0]]
n_gt = gt_crowd_np.size if class_slice is None else class_slice[1].size
if n_gt == 0:
return _unmatched_contribution(p_scores_np)
assert bbox_iou_matrix_np is not None, (
"the image-wide box_iou matrix is built whenever the image has both detections and GTs, "
"and this class has at least one of each"
)
order = np.argsort(-p_scores_np, kind="stable")
if class_slice is None:
iou_matrix_sorted, class_crowd_np = bbox_iou_matrix_np[order], gt_crowd_np
else:
pred_indices, gt_indices = class_slice
iou_matrix_sorted = bbox_iou_matrix_np[np.ix_(pred_indices[order], gt_indices)]
class_crowd_np = gt_crowd_np[gt_indices]
matches_np, ignore_np, total_gt = _match_sorted_iou_matrix(iou_matrix_sorted, class_crowd_np, iou_threshold)
return p_scores_np[order].astype(np.float32, copy=False), matches_np, ignore_np, total_gt
def _accumulate_segm_class(
preds: dict[str, Tensor],
targets: dict[str, Tensor],
gt_crowd: Tensor,
class_id: int,
n_gt: int,
iou_threshold: float,
) -> _ClassContribution:
"""Accumulate one class of one image on boolean-mask IoU.
Mask IoU stays class-local rather than being shared image-wide like the box path: an all-pairs
mask matrix is much larger, and most of it would cover pairs that cannot match.
Args:
preds: One image's predictions, keyed as documented on ``build_matching_data()``.
targets: One image's ground truths, keyed as documented on ``build_matching_data()``.
gt_crowd: Bool tensor [M] aligned to ``targets["labels"]``, already validated by the caller.
class_id: Class to accumulate.
n_gt: Number of GTs of *class_id* in this image. Taken from the caller's host-side label
counts rather than read back out of ``targets["labels"]``, which would cost one
device-to-host sync per class.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Tuple ``(scores_np, matches_np, ignore_np, total_gt)`` contributed by *class_id*.
Raises:
ValueError: If ``masks`` is missing from *preds* or *targets*. A class with no GT of its own
returns before that lookup, so a one-sided class is never reported.
"""
pred_mask_c = preds["labels"] == class_id
p_scores = preds["scores"][pred_mask_c]
if n_gt == 0:
# TODO: support bfloat16 natively once numpy adds bf16 dtype
return _unmatched_contribution(np.asarray(p_scores.detach().float().cpu().numpy(), dtype=np.float32))
pred_masks = preds.get("masks")
gt_masks = targets.get("masks")
if pred_masks is None or gt_masks is None:
raise ValueError("iou_type='segm' requires 'masks' in both preds and targets")
gt_mask_c = targets["labels"] == class_id
return _match_single_class_segm(
p_scores, pred_masks[pred_mask_c], gt_masks[gt_mask_c], gt_crowd[gt_mask_c], iou_threshold
)
def build_matching_data(
preds_list: list[dict[str, Tensor]],
targets_list: list[dict[str, Tensor]],
@@ -156,6 +438,11 @@ def build_matching_data(
directly to ``merge_matching_data()`` and ultimately consumed by ``sweep_confidence_thresholds()`` after conversion
to list form.
Detections are ranked in the dtype of ``preds["scores"]``, so a ``float64`` input keeps the full precision that
separates near-tied scores, and detections that really are tied keep their input order. Both ``iou_type`` paths
share that rule, which makes the TP/FP split reproducible across devices and dtypes. The returned scores are
float32 either way.
Args:
preds_list: Per-image predictions. Each dict must contain:
@@ -195,11 +482,9 @@ def build_matching_data(
pred_boxes = preds["boxes"] # [N, 4]
pred_scores = preds["scores"] # [N]
pred_labels = preds["labels"] # [N]
pred_masks = preds.get("masks") # [N, H, W] | None
gt_boxes = targets["boxes"] # [M, 4]
gt_labels = targets["labels"] # [M]
gt_masks = targets.get("masks") # [M, H, W] | None
raw_crowd = targets.get(
"iscrowd",
torch.zeros(len(gt_labels), dtype=torch.long, device=gt_labels.device),
@@ -225,52 +510,69 @@ def build_matching_data(
gt_noncrowd_count = Counter(label for label, crowd in zip(gt_label_ids, gt_crowd_ids) if not crowd)
all_class_ids: set[int] = set(gt_count) | set(pred_count)
for class_id in all_class_ids:
n_pred = pred_count.get(class_id, 0)
n_gt = gt_count.get(class_id, 0)
# One image whose detections and GTs are all of the same class needs no per-class positions
# at all: that class owns every row and column of the IoU matrix below.
single_class = len(all_class_ids) == 1
pred_groups: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {}
gt_groups: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {}
if iou_type == "bbox" and pred_count and not single_class:
pred_groups = _group_indices_by_label(np.asarray(pred_label_ids, dtype=np.int64))
gt_groups = _group_indices_by_label(np.asarray(gt_label_ids, dtype=np.int64))
gt_crowd_np = np.asarray(gt_crowd_ids, dtype=np.bool_)
bbox_iou_matrix_np: np.ndarray[Any, np.dtype[np.float32]] | None = None
# Left in the caller's own dtype: this array orders the greedy loop, and a float32 cast
# taken here would collapse near-tied float64 scores onto one value and hand the GT to
# whichever detection happened to come first instead of to the higher-scoring one. Only
# bfloat16 is cast, because numpy has no bfloat16 dtype. The scores that are handed back
# are cast to float32 after the ordering is fixed, so the returned dtype is unchanged.
pred_scores_np: np.ndarray[Any, np.dtype[np.floating[Any]]] | None = None
if iou_type == "bbox" and pred_count:
pred_scores_cpu = pred_scores.detach().cpu()
if pred_scores_cpu.dtype == torch.bfloat16:
pred_scores_cpu = pred_scores_cpu.float()
pred_scores_np = pred_scores_cpu.numpy()
if iou_type == "bbox" and pred_count and gt_count:
# One image-wide GPU IoU operation and one matrix transfer replace the
# class-local launches/transfers. Host slicing below retains class isolation.
bbox_iou_matrix_np = np.asarray(
box_iou(pred_boxes, gt_boxes).detach().float().cpu().numpy(), dtype=np.float32
)
for class_id in all_class_ids:
entry = acc.setdefault(
class_id,
{"scores": [], "matches": [], "ignore": [], "total_gt": 0},
)
if n_pred == 0:
if pred_count.get(class_id, 0) == 0:
entry["total_gt"] = cast(int, entry["total_gt"]) + gt_noncrowd_count.get(class_id, 0)
continue
# Only materialize the boolean mask / gather predictions for classes that actually
# have detections — gt_mask_c below is deferred further, to classes that also matter.
pred_mask_c = pred_labels == class_id
p_scores = pred_scores[pred_mask_c]
if n_gt == 0:
# TODO: support bfloat16 natively once numpy adds bf16 dtype
sc = np.asarray(p_scores.float().cpu().numpy(), dtype=np.float32)
order = np.argsort(-sc)
cast(list[float], entry["scores"]).extend(sc[order].tolist())
cast(list[int], entry["matches"]).extend([0] * n_pred)
cast(list[bool], entry["ignore"]).extend([False] * n_pred)
continue
gt_mask_c = gt_labels == class_id
gt_crowd_c = gt_crowd[gt_mask_c]
# The one place ``iou_type`` is branched on per class: each helper owns its own scores
# variable and its own empty-GT case, so the two paths cannot drift out of agreement.
if iou_type == "bbox":
p_items: Tensor = pred_boxes[pred_mask_c] # [n_pred, 4]
gt_items: Tensor = gt_boxes[gt_mask_c] # [n_gt, 4]
assert pred_scores_np is not None, (
"the image's scores are moved to host whenever it has detections, and this class has one"
)
class_slice = None if single_class else (pred_groups[class_id], gt_groups.get(class_id, _NO_INDICES))
scores_np, matches_np, ignore_np, total_gt = _accumulate_bbox_class(
pred_scores_np,
class_slice,
gt_crowd_np,
bbox_iou_matrix_np,
iou_threshold,
)
else:
if pred_masks is None or gt_masks is None:
raise ValueError("iou_type='segm' requires 'masks' in both preds and targets")
p_items = pred_masks[pred_mask_c] # [n_pred, H, W]
gt_items = gt_masks[gt_mask_c] # [n_gt, H, W]
scores_np, matches_np, ignore_np, total_gt = _accumulate_segm_class(
preds, targets, gt_crowd, class_id, gt_count.get(class_id, 0), iou_threshold
)
scores_np, matches_np, ignore_np, total_gt = _match_single_class(
p_scores, p_items, gt_items, gt_crowd_c, iou_threshold, iou_type
)
cast(list[float], entry["scores"]).extend(float(score) for score in scores_np)
cast(list[int], entry["matches"]).extend(int(match) for match in matches_np)
cast(list[bool], entry["ignore"]).extend(bool(ignore) for ignore in ignore_np)
# ndarray.tolist() converts in C and yields the same Python scalars a per-element
# float()/int()/bool() generator would, one interpreter round-trip instead of N.
cast(list[float], entry["scores"]).extend(cast(list[float], scores_np.tolist()))
cast(list[int], entry["matches"]).extend(cast(list[int], matches_np.tolist()))
cast(list[bool], entry["ignore"]).extend(cast(list[bool], ignore_np.tolist()))
entry["total_gt"] = cast(int, entry["total_gt"]) + total_gt
return {
tests/evaluation/test_evaluation.py
+588 -102
View File
@@ -15,7 +15,10 @@ from torchvision.ops import box_iou
from rfdetr.evaluation.matching import (
_compute_mask_iou,
_match_single_class,
_match_rows_with_crowd,
_match_rows_without_crowd,
_match_single_class_segm,
_match_sorted_iou_matrix,
build_matching_data,
distributed_merge_matching_data,
init_matching_accumulator,
@@ -76,50 +79,255 @@ class TestComputeMaskIou:
# ---------------------------------------------------------------------------
# _match_single_class
# _match_sorted_iou_matrix
# ---------------------------------------------------------------------------
class TestMatchSingleClass:
"""Unit tests for the private _match_single_class helper."""
class TestMatchSortedIouMatrix:
"""Direct unit tests for the private _match_sorted_iou_matrix helper.
Every other test in this file drives this function indirectly, through ``_match_single_class_segm``,
``_accumulate_bbox_class``, or ``build_matching_data``. These cases call it directly with small hand-built numpy IoU
matrices, so a regression here is localized to this one function instead of surfacing only as a mismatch several
call frames up.
"""
def test_disjoint_and_matched_gts_split_into_tp_and_fp(self) -> None:
"""One detection with sufficient IoU is a TP; one with insufficient IoU is a FP.
Two detections, each with its own GT (no competition between them), isolates the
threshold comparison from the greedy claiming logic: row 0's IoU (0.9) clears the 0.5
threshold against its own GT, row 1's IoU (0.1) does not clear it against its own GT.
"""
iou_matrix_sorted = np.array([[0.9, 0.0], [0.0, 0.1]], dtype=np.float32)
gt_crowd_np = np.zeros(2, dtype=np.bool_)
matches, ignore, total_gt = _match_sorted_iou_matrix(iou_matrix_sorted, gt_crowd_np, iou_threshold=0.5)
np.testing.assert_array_equal(matches, [1, 0])
np.testing.assert_array_equal(ignore, [False, False])
assert total_gt == 2
def test_detection_matched_only_to_crowd_gt_is_ignored_not_fp(self) -> None:
"""A detection whose only sufficient-IoU GT is a crowd instance is ignored, not a FP.
One detection, one crowd GT, IoU above threshold: the crowd-ignore branch must fire instead of the false-
positive branch, and the crowd GT must not inflate total_gt.
"""
iou_matrix_sorted = np.array([[0.9]], dtype=np.float32)
gt_crowd_np = np.array([True], dtype=np.bool_)
matches, ignore, total_gt = _match_sorted_iou_matrix(iou_matrix_sorted, gt_crowd_np, iou_threshold=0.5)
np.testing.assert_array_equal(matches, [0])
np.testing.assert_array_equal(ignore, [True])
assert total_gt == 0
def test_zero_gt_columns_raises_value_error(self) -> None:
"""An IoU matrix with no GT columns (m=0) raises ValueError, not a silent empty result.
``np.argmax`` on the per-detection IoU row is unconditional, so a matrix with zero columns fails inside the loop
instead of returning early. Both call sites (``_accumulate_bbox_class``, ``_match_single_class_segm``) guard
against calling this function when a class has no GT, so this test documents/locks the current contract rather
than exercising a path either caller actually reaches.
"""
iou_matrix_sorted = np.zeros((2, 0), dtype=np.float32)
gt_crowd_np = np.zeros(0, dtype=np.bool_)
with pytest.raises(ValueError, match="argmax of an empty sequence"):
_match_sorted_iou_matrix(iou_matrix_sorted, gt_crowd_np, iou_threshold=0.5)
def test_zero_pred_rows_returns_empty_arrays_without_error(self) -> None:
"""An IoU matrix with no prediction rows (n=0) returns empty arrays, unlike the m=0 case.
The greedy loop iterates ``range(len(iou_matrix_sorted))``, which is 0 when there are no predictions, so the
loop body — and the ``np.argmax`` call that raises for m=0 — never executes. ``total_gt`` is still computed from
``gt_crowd_np`` alone, so it is asserted against a non-trivial mix of crowd and non-crowd GTs rather than an
all-non-crowd matrix.
"""
iou_matrix_sorted = np.zeros((0, 2), dtype=np.float32)
gt_crowd_np = np.array([False, True], dtype=np.bool_)
matches, ignore, total_gt = _match_sorted_iou_matrix(iou_matrix_sorted, gt_crowd_np, iou_threshold=0.5)
assert matches.shape == (0,)
assert ignore.shape == (0,)
assert total_gt == 1
@pytest.mark.parametrize("seed", [pytest.param(seed, id=f"seed-{seed}") for seed in range(20)])
def test_crowd_free_loop_agrees_with_the_general_loop(self, seed: int) -> None:
"""``_match_rows_without_crowd`` returns exactly what ``_match_rows_with_crowd`` returns on a crowd-free class.
``_match_sorted_iou_matrix`` picks between the two loops on ``gt_crowd_np.any()``, so no other test in this
suite can observe the equivalence they are built on: a crowd-free input never reaches the general loop again.
Each seed draws a dense IoU matrix with six times as many detections as GTs and matches at a 0.5 threshold, so
GTs are contested and the crowd-free loop's fallback (its best GT already claimed by a higher-scoring
detection) is reached instead of only its two shortcut branches — the assertion that every GT ends up claimed
is what pins that contention down.
"""
rng = np.random.default_rng(seed)
iou_matrix_sorted = rng.uniform(0.0, 1.0, size=(24, 4)).astype(np.float32)
gt_crowd_np = np.zeros(4, dtype=np.bool_)
crowd_free = _match_rows_without_crowd(iou_matrix_sorted, iou_threshold=0.5)
general = _match_rows_with_crowd(iou_matrix_sorted, gt_crowd_np, iou_threshold=0.5)
np.testing.assert_array_equal(crowd_free[0], general[0])
np.testing.assert_array_equal(crowd_free[1], general[1])
assert crowd_free[2] == general[2]
assert crowd_free[0].sum() == 4
# ---------------------------------------------------------------------------
# Mask rasterisation shared by the segm matcher tests and build_matching_data
# ---------------------------------------------------------------------------
def _masks_from_boxes(boxes: list[list[float]], height: int, width: int) -> torch.Tensor:
"""Rasterise integer-aligned xyxy boxes into a boolean mask stack.
Lets the ``iou_type="segm"`` path be driven by the same geometry as the ``"bbox"`` path: for
integer-aligned boxes the pixel-count mask IoU equals the box-area IoU exactly, so both paths
see the same IoU matrix and any difference in the result comes from the ranking alone.
Args:
boxes: Rows of ``[x1, y1, x2, y2]`` in pixel coordinates.
height: Height of each rasterised mask.
width: Width of each rasterised mask.
Returns:
Boolean tensor of shape ``[len(boxes), height, width]``.
Examples:
>>> _masks_from_boxes([[0, 0, 2, 1]], height=1, width=4).int().tolist()
[[[1, 1, 0, 0]]]
"""
masks = torch.zeros(len(boxes), height, width, dtype=torch.bool)
for index, (x1, y1, x2, y2) in enumerate(boxes):
masks[index, int(y1) : int(y2), int(x1) : int(x2)] = True
return masks
def _reference_greedy_match(
order: list[int],
iou_matrix: torch.Tensor,
gt_crowd: torch.Tensor,
iou_threshold: float,
) -> tuple[np.ndarray, np.ndarray, int]:
"""Naive, independently-written greedy matcher used as a ground-truth oracle.
Reimplements the COCO greedy-matching contract directly from spec with plain Python loops and
no shared code with ``_match_single_class_segm``/``_match_sorted_iou_matrix`` (no numpy
vectorization, no hoisted crowd mask) — a divergence in the optimized/numpy-ported
implementation (e.g. a tie-break or crowd-handling regression) will disagree with this
reference instead of silently agreeing with itself. Module-level because it is shared by
``TestMatchSingleClassSegm`` (segm path, one class at a time) and
``TestBuildMatchingDataBboxDifferential`` (bbox path, one class-slice at a time).
Args:
order: Detection indices into ``iou_matrix``'s rows, in descending-score order.
iou_matrix: Pairwise IoU, ``[n_preds, n_gt]``, rows in original (unsorted) order.
gt_crowd: Bool tensor ``[n_gt]``, ``True`` for crowd instances.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Tuple ``(matches, ignore, total_gt)`` aligned to ``order``.
Examples:
>>> order = [0, 1]
>>> iou_matrix = torch.tensor([[0.9, 0.0], [0.0, 0.9]])
>>> gt_crowd = torch.zeros(2, dtype=torch.bool)
>>> matches, ignore, total_gt = _reference_greedy_match(order, iou_matrix, gt_crowd, 0.5)
>>> matches.tolist()
[1, 1]
>>> total_gt
2
"""
n_gt = iou_matrix.shape[1]
gt_matched = [False] * n_gt
matches = np.zeros(len(order), dtype=np.int64)
ignore = np.zeros(len(order), dtype=np.bool_)
for out_i, orig_i in enumerate(order):
best_iou, best_gt = -1.0, -1
for j in range(n_gt):
if gt_crowd[j] or gt_matched[j]:
continue
iou = float(iou_matrix[orig_i, j])
if iou > best_iou:
best_iou, best_gt = iou, j
if best_gt != -1 and best_iou >= iou_threshold:
matches[out_i] = 1
gt_matched[best_gt] = True
continue
for j in range(n_gt):
if gt_crowd[j] and float(iou_matrix[orig_i, j]) >= iou_threshold:
ignore[out_i] = True
break
total_gt = int((~gt_crowd.numpy()).sum())
return matches, ignore, total_gt
# ---------------------------------------------------------------------------
# _match_single_class_segm
# ---------------------------------------------------------------------------
class TestMatchSingleClassSegm:
"""Unit tests for the private _match_single_class_segm helper.
Cases are written as xyxy boxes and rasterised with ``_masks_from_boxes``, because for integer-aligned boxes the
pixel-count mask IoU equals the box-area IoU exactly — the expected IoU of each case can be read straight off the
coordinates. ``test_uses_mask_overlap_not_bounding_box`` is the deliberate exception, and builds masks that no box
could describe.
"""
#: Side of the square canvas every box below is rasterised onto — larger than any coordinate used.
_CANVAS = 64
@classmethod
def _mask(cls, *coords: float) -> torch.Tensor:
"""Return a [1, H, W] boolean mask covering the single xyxy box *coords*."""
return _masks_from_boxes([list(coords)], cls._CANVAS, cls._CANVAS)
@classmethod
def _masks(cls, *rows: list[float]) -> torch.Tensor:
"""Return an [N, H, W] boolean mask stack from a sequence of [x1,y1,x2,y2] rows."""
return _masks_from_boxes(list(rows), cls._CANVAS, cls._CANVAS)
@staticmethod
def _box(*coords: float) -> torch.Tensor:
"""Return a [1, 4] float32 box tensor from (x1, y1, x2, y2)."""
return torch.tensor([list(coords)], dtype=torch.float32)
@staticmethod
def _boxes(*rows: list[float]) -> torch.Tensor:
"""Return an [N, 4] float32 tensor from a sequence of [x1,y1,x2,y2] rows."""
return torch.tensor(list(rows), dtype=torch.float32)
def _random_boxes(count: int, canvas: int) -> torch.Tensor:
"""Return *count* random integer-aligned xyxy boxes that fit inside a *canvas*-square image."""
top_left = torch.randint(0, canvas // 2, (count, 2))
extent = torch.randint(1, canvas // 2 + 1, (count, 2))
return torch.cat([top_left, top_left + extent], dim=1).float()
def _run(
self,
pred_scores: torch.Tensor,
pred_items: torch.Tensor,
gt_items: torch.Tensor,
pred_masks: torch.Tensor,
gt_masks: torch.Tensor,
gt_crowd: torch.Tensor | None = None,
iou_threshold: float = 0.5,
iou_type: str = "bbox",
) -> tuple[np.ndarray, np.ndarray, np.ndarray, int]:
"""Run the matcher, defaulting every GT to non-crowd."""
if gt_crowd is None:
gt_crowd = torch.zeros(len(gt_items), dtype=torch.bool)
return _match_single_class(pred_scores, pred_items, gt_items, gt_crowd, iou_threshold, iou_type)
gt_crowd = torch.zeros(len(gt_masks), dtype=torch.bool)
return _match_single_class_segm(pred_scores, pred_masks, gt_masks, gt_crowd, iou_threshold)
def test_perfect_overlap_is_tp(self) -> None:
"""A prediction that perfectly overlaps the GT box is a true positive."""
"""A prediction whose mask is identical to the GT mask is a true positive."""
scores = torch.tensor([0.9])
box = self._box(0, 0, 10, 10)
_, matches, ignore, total_gt = self._run(scores, box, box)
mask = self._mask(0, 0, 10, 10)
_, matches, ignore, total_gt = self._run(scores, mask, mask)
assert matches[0] == 1
assert not ignore[0]
assert total_gt == 1
def test_disjoint_box_is_fp(self) -> None:
"""A prediction with no overlap with the GT box is a false positive."""
def test_disjoint_mask_is_fp(self) -> None:
"""A prediction with no overlap with the GT mask is a false positive."""
scores = torch.tensor([0.9])
pred = self._box(0, 0, 10, 10)
gt = self._box(50, 50, 60, 60)
pred = self._mask(0, 0, 10, 10)
gt = self._mask(50, 50, 60, 60)
_, matches, ignore, total_gt = self._run(scores, pred, gt)
assert matches[0] == 0
assert not ignore[0]
@@ -128,8 +336,8 @@ class TestMatchSingleClass:
def test_iou_below_threshold_is_fp(self) -> None:
"""A detection with IoU < threshold must be marked as FP."""
scores = torch.tensor([0.9])
pred = self._box(0, 0, 5, 10) # area = 50
gt = self._box(6, 0, 10, 10) # area = 40 — no overlap
pred = self._mask(0, 0, 5, 10) # area = 50
gt = self._mask(6, 0, 10, 10) # area = 40 — no overlap
_, matches, _, _ = self._run(scores, pred, gt, iou_threshold=0.5)
assert matches[0] == 0
@@ -137,8 +345,8 @@ class TestMatchSingleClass:
"""When two predictions compete for one GT, the higher-score pred wins."""
# Sorted descending: [0.9, 0.5] -> first gets TP, second gets FP.
scores = torch.tensor([0.5, 0.9])
preds = self._boxes([0, 0, 10, 10], [0, 0, 10, 10])
gt = self._box(0, 0, 10, 10)
preds = self._masks([0, 0, 10, 10], [0, 0, 10, 10])
gt = self._mask(0, 0, 10, 10)
scores_out, matches, _, _ = self._run(scores, preds, gt)
assert list(scores_out) == pytest.approx([0.9, 0.5])
assert matches[0] == 1 # highest score -> TP
@@ -147,9 +355,9 @@ class TestMatchSingleClass:
def test_crowd_gt_match_is_ignored_not_fp(self) -> None:
"""A detection matched to a crowd GT is ignored, not a false positive."""
scores = torch.tensor([0.9])
box = self._box(0, 0, 10, 10)
mask = self._mask(0, 0, 10, 10)
gt_crowd = torch.tensor([True])
_, matches, ignore, total_gt = self._run(scores, box, box, gt_crowd=gt_crowd)
_, matches, ignore, total_gt = self._run(scores, mask, mask, gt_crowd=gt_crowd)
assert matches[0] == 0 # not TP
assert ignore[0] # ignored -> not counted as FP
assert total_gt == 0 # crowd GT excluded from denominator
@@ -157,95 +365,60 @@ class TestMatchSingleClass:
def test_non_crowd_gt_counts_in_total_gt(self) -> None:
"""Non-crowd GTs are counted in total_gt."""
scores = torch.tensor([0.9])
box = self._box(0, 0, 10, 10)
mask = self._mask(0, 0, 10, 10)
gt_crowd = torch.tensor([False])
_, _, _, total_gt = self._run(scores, box, box, gt_crowd=gt_crowd)
_, _, _, total_gt = self._run(scores, mask, mask, gt_crowd=gt_crowd)
assert total_gt == 1
def test_mixed_crowd_only_non_crowd_in_total_gt(self) -> None:
"""Only non-crowd instances contribute to total_gt."""
scores = torch.tensor([0.9])
pred = self._box(0, 0, 5, 5) # overlaps neither GT significantly
gt_boxes = self._boxes([0, 0, 10, 10], [20, 20, 30, 30])
pred = self._mask(0, 0, 5, 5) # overlaps neither GT significantly
gt_masks = self._masks([0, 0, 10, 10], [20, 20, 30, 30])
gt_crowd = torch.tensor([False, True]) # second GT is crowd
_, _, _, total_gt = self._run(scores, pred, gt_boxes, gt_crowd=gt_crowd)
_, _, _, total_gt = self._run(scores, pred, gt_masks, gt_crowd=gt_crowd)
assert total_gt == 1
def test_scores_returned_in_descending_order(self) -> None:
"""Output scores must be sorted in descending order."""
scores = torch.tensor([0.3, 0.9, 0.6])
preds = self._boxes([0, 0, 10, 10], [20, 20, 30, 30], [40, 40, 50, 50])
gt = self._box(20, 20, 30, 30)
preds = self._masks([0, 0, 10, 10], [20, 20, 30, 30], [40, 40, 50, 50])
gt = self._mask(20, 20, 30, 30)
scores_out, _, _, _ = self._run(scores, preds, gt)
assert list(scores_out) == pytest.approx([0.9, 0.6, 0.3])
def test_segm_iou_type_identical_masks_is_tp(self) -> None:
"""Identical masks with iou_type='segm' should yield a TP."""
mask = torch.ones(1, 4, 4, dtype=torch.bool)
scores = torch.tensor([0.9])
gt_crowd = torch.tensor([False])
_, matches, _, total_gt = _match_single_class(scores, mask, mask, gt_crowd, 0.5, "segm")
assert matches[0] == 1
assert total_gt == 1
def test_uses_mask_overlap_not_bounding_box(self) -> None:
"""Matching is driven by pixel overlap, not by the region the masks span.
@staticmethod
def _reference_greedy_match(
order: list[int],
iou_matrix: torch.Tensor,
gt_crowd: torch.Tensor,
iou_threshold: float,
) -> tuple[np.ndarray, np.ndarray, int]:
"""Naive, independently-written greedy matcher used as a ground-truth oracle.
Reimplements the COCO greedy-matching contract directly from spec with plain Python
loops and no shared code with ``_match_single_class`` (no numpy vectorization, no
hoisted crowd mask) — a divergence in the optimized/numpy-ported implementation (e.g. a
tie-break or crowd-handling regression) will disagree with this reference instead of
silently agreeing with itself.
Args:
order: Detection indices into ``iou_matrix``'s rows, in descending-score order.
iou_matrix: Pairwise IoU, ``[n_preds, n_gt]``, rows in original (unsorted) order.
gt_crowd: Bool tensor ``[n_gt]``, ``True`` for crowd instances.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Tuple ``(matches, ignore, total_gt)`` aligned to ``order``.
Two interleaved comb masks share no pixel at all (mask IoU 0.0), yet the boxes bounding them overlap at exactly
the 0.5 threshold — geometry that a box-area proxy would score as a TP and true mask overlap scores as a FP.
This is the one case in the class whose masks are built directly rather than rasterised from boxes, because no
box can describe them.
"""
n_gt = iou_matrix.shape[1]
gt_matched = [False] * n_gt
matches = np.zeros(len(order), dtype=np.int64)
ignore = np.zeros(len(order), dtype=np.bool_)
for out_i, orig_i in enumerate(order):
best_iou, best_gt = -1.0, -1
for j in range(n_gt):
if gt_crowd[j] or gt_matched[j]:
continue
iou = float(iou_matrix[orig_i, j])
if iou > best_iou:
best_iou, best_gt = iou, j
if best_gt != -1 and best_iou >= iou_threshold:
matches[out_i] = 1
gt_matched[best_gt] = True
continue
for j in range(n_gt):
if gt_crowd[j] and float(iou_matrix[orig_i, j]) >= iou_threshold:
ignore[out_i] = True
break
total_gt = int((~gt_crowd.numpy()).sum())
return matches, ignore, total_gt
pred = torch.zeros(1, 4, 4, dtype=torch.bool)
pred[0, :, ::2] = True # columns 0 and 2 -> bounding box (0, 0, 3, 4)
gt = ~pred # columns 1 and 3 -> bounding box (1, 0, 4, 4), box IoU 8/16 = 0.5
scores = torch.tensor([0.9])
_, matches, ignore, total_gt = self._run(scores, pred, gt)
assert matches[0] == 0
assert not ignore[0]
assert total_gt == 1
def test_greedy_loop_does_not_sync_a_tensor_per_detection(self) -> None:
"""Regression test for #416: the per-detection greedy loop must not force one device-to-host tensor sync
(``Tensor.__bool__``) per detection, and the numpy-ported matching output must agree with an independent
reference implementation of the same algorithm at scale (matches/ignore/total_gt) — the sync-count assert alone
cannot catch a logic regression in the torch->numpy port (e.g. a tie-break or crowd-handling change)."""
n, m = 50, 10
cannot catch a logic regression in the torch->numpy port (e.g. a tie-break or crowd-handling change).
The geometry is random but integer-aligned, so ``box_iou`` over the source boxes reproduces the mask IoU the
matcher computes exactly, and the oracle stays independent of ``_compute_mask_iou``.
"""
n, m, canvas = 50, 10, 32
scores = torch.rand(n)
preds = torch.rand(n, 4) * 100
preds[:, 2:] += preds[:, :2] + 1.0
gts = torch.rand(m, 4) * 100
gts[:, 2:] += gts[:, :2] + 1.0
pred_boxes = self._random_boxes(n, canvas)
gt_boxes = self._random_boxes(m, canvas)
pred_masks = _masks_from_boxes(pred_boxes.tolist(), canvas, canvas)
gt_masks = _masks_from_boxes(gt_boxes.tolist(), canvas, canvas)
gt_crowd = torch.zeros(m, dtype=torch.bool)
gt_crowd[:3] = True # exercise the crowd/ignore branch at scale, not just n<=3 cases
@@ -258,17 +431,17 @@ class TestMatchSingleClass:
return orig_bool(self)
with patch.object(torch.Tensor, "__bool__", counting_bool):
scores_out, matches, ignore, total_gt = self._run(scores, preds, gts, gt_crowd=gt_crowd)
scores_out, matches, ignore, total_gt = self._run(scores, pred_masks, gt_masks, gt_crowd=gt_crowd)
assert call_count < n, (
f"_match_single_class triggered {call_count} tensor->bool syncs for n={n} "
f"_match_single_class_segm triggered {call_count} tensor->bool syncs for n={n} "
"detections; expected O(1) syncs, not O(n) — the greedy loop should operate "
"on host data, not per-iteration device tensor comparisons"
)
order = torch.argsort(scores, descending=True).tolist()
iou_matrix = box_iou(preds, gts)
ref_matches, ref_ignore, ref_total_gt = self._reference_greedy_match(order, iou_matrix, gt_crowd, 0.5)
iou_matrix = box_iou(pred_boxes, gt_boxes)
ref_matches, ref_ignore, ref_total_gt = _reference_greedy_match(order, iou_matrix, gt_crowd, 0.5)
assert list(scores_out) == pytest.approx(scores[order].tolist())
assert np.array_equal(matches, ref_matches)
assert np.array_equal(ignore, ref_ignore)
@@ -279,6 +452,24 @@ class TestMatchSingleClass:
# build_matching_data
# ---------------------------------------------------------------------------
# Geometry whose greedy matching outcome depends on the order the two detections are ranked in.
# The two GTs overlap; the narrow detection can only reach the first, the wide one can reach either:
#
# IoU(narrow, GT-A) = 0.700 IoU(narrow, GT-B) = 0.133
# IoU(wide, GT-A) = 0.667 IoU(wide, GT-B) = 0.538
#
# Ranking the narrow detection first yields two TPs (it claims GT-A, the wide one falls back to
# GT-B); ranking the wide one first yields one (it claims GT-A, and GT-B is out of the narrow
# detection's reach).
_ORDER_SENSITIVE_GT_BOXES = [[0, 0, 100, 10], [50, 0, 150, 10]]
_NARROW_PRED_BOX = [0, 0, 70, 10]
_WIDE_PRED_BOX = [20, 0, 120, 10]
# Padding that overlaps neither GT, so it cannot change the outcome, and pushes the detection count
# past the ~32-element cutoff above which torch's default (unstable) sort permutes tied scores.
_FILLER_PRED_BOX = [200, 0, 210, 10]
_NUM_FILLER_PREDS = 38
_ORDER_SENSITIVE_MASK_SIZE = (10, 210)
class TestBuildMatchingData:
"""Unit tests for build_matching_data()."""
@@ -289,10 +480,11 @@ class TestBuildMatchingData:
scores: list,
labels: list,
masks: torch.Tensor | None = None,
scores_dtype: torch.dtype = torch.float32,
) -> dict[str, torch.Tensor]:
d: dict[str, torch.Tensor] = {
"boxes": torch.tensor(boxes, dtype=torch.float32).reshape(-1, 4),
"scores": torch.tensor(scores, dtype=torch.float32),
"scores": torch.tensor(scores, dtype=scores_dtype),
"labels": torch.tensor(labels, dtype=torch.int64),
}
if masks is not None:
@@ -366,6 +558,78 @@ class TestBuildMatchingData:
assert result[0]["total_gt"] == 1
assert result[1]["total_gt"] == 1
def test_bbox_iou_is_computed_once_per_image(self) -> None:
"""BBox matching shares one image-wide IoU matrix while preserving class-local matches."""
pred = self._make_pred(
[[0, 0, 10, 10], [0, 0, 10, 10], [40, 40, 50, 50]],
[0.8, 0.95, 0.9],
[0, 1, 0],
)
target = self._make_target([[0, 0, 10, 10], [20, 20, 30, 30]], [0, 1])
with patch("rfdetr.evaluation.matching.box_iou", wraps=box_iou) as box_iou_spy:
result = build_matching_data([pred], [target])
box_iou_spy.assert_called_once_with(pred["boxes"], target["boxes"])
np.testing.assert_allclose(result[0]["scores"], [0.9, 0.8], rtol=1e-6)
np.testing.assert_array_equal(result[0]["matches"], [0, 1])
np.testing.assert_array_equal(result[1]["matches"], [0])
assert result[0]["total_gt"] == 1
assert result[1]["total_gt"] == 1
@pytest.mark.parametrize("iou_type", [pytest.param("bbox", id="bbox"), pytest.param("segm", id="segm")])
def test_tied_scores_are_ranked_in_input_order_on_both_iou_types(self, iou_type: str) -> None:
"""Detections tied on score are ranked in input order on the bbox path and the segm path alike.
The two paths ran through one matcher before the per-image IoU matrix was introduced, so
they broke ties identically; they must still agree afterwards. This is the scenario that
makes a divergence visible: 40 same-class detections, the two order-sensitive ones tied at
0.5 and the padding scoring above them, which places the tie group where ``torch.argsort``'s
default unstable sort reverses it and ``np.argsort(kind="stable")`` does not. Ranked in
input order the narrow detection claims GT-A and the wide one falls back to GT-B; reversed,
the wide detection takes GT-A and the narrow one is left with a GT it cannot reach.
"""
boxes = [_NARROW_PRED_BOX, _WIDE_PRED_BOX, *([_FILLER_PRED_BOX] * _NUM_FILLER_PREDS)]
scores = [0.5, 0.5, *([0.9] * _NUM_FILLER_PREDS)]
pred = self._make_pred(
boxes,
scores,
[0] * len(scores),
masks=_masks_from_boxes(boxes, *_ORDER_SENSITIVE_MASK_SIZE),
)
target = self._make_target(
_ORDER_SENSITIVE_GT_BOXES,
[0, 0],
masks=_masks_from_boxes(_ORDER_SENSITIVE_GT_BOXES, *_ORDER_SENSITIVE_MASK_SIZE),
)
result = build_matching_data([pred], [target], iou_type=iou_type)
# Output is in descending-score order, so the higher-scoring padding comes first.
np.testing.assert_array_equal(result[0]["matches"], [0] * _NUM_FILLER_PREDS + [1, 1])
assert result[0]["total_gt"] == 2
def test_near_tied_float64_scores_are_ranked_at_full_precision(self) -> None:
"""Float64 scores that collapse onto one float32 value still rank by their full precision.
``0.50000001`` and ``0.50000002`` are distinct in float64 and the same number in float32, so
casting to float32 before the ranking hands greedy priority to whichever detection came
first in the input instead of to the higher-scoring one. Here the wide detection is first in
the input but scores lower: ranked at full precision the narrow detection claims GT-A and
the wide one falls back to GT-B, while a collapsed ranking gives GT-A to the wide detection
and leaves the narrow one nothing to match. The returned scores stay float32 either way —
only the ordering is allowed to see the caller's dtype.
"""
boxes = [_WIDE_PRED_BOX, _NARROW_PRED_BOX, *([_FILLER_PRED_BOX] * _NUM_FILLER_PREDS)]
scores = [0.50000001, 0.50000002, *([0.5] * _NUM_FILLER_PREDS)]
pred = self._make_pred(boxes, scores, [0] * len(scores), scores_dtype=torch.float64)
target = self._make_target(_ORDER_SENSITIVE_GT_BOXES, [0, 0])
result = build_matching_data([pred], [target])
np.testing.assert_array_equal(result[0]["matches"], [1, 1, *([0] * _NUM_FILLER_PREDS)])
assert result[0]["scores"].dtype == np.float32
def test_multi_image_batch_accumulates(self) -> None:
"""Two-image batch must concatenate scores and sum total_gt."""
pred1 = self._make_pred([[0, 0, 10, 10]], [0.9], [0])
@@ -504,7 +768,7 @@ class TestBuildMatchingData:
2. the bulk reads that remain (``tolist``) are exactly three per image — the pred labels, the
GT labels and ``iscrowd`` — and do not grow with the class count.
Not covered: ``_match_single_class`` still moves its own scores and IoUs to host with
Not covered: ``_match_single_class_segm`` still moves its own scores and IoUs to host with
``.cpu()/.numpy()`` once per class that has detections. That is pre-existing and untouched
here; this test fixes the cost of the classes that never reach the matcher.
"""
@@ -550,6 +814,228 @@ class TestBuildMatchingData:
assert result[class_id]["total_gt"] == expected_total_gt
# ---------------------------------------------------------------------------
# Randomized differential oracle for build_matching_data(iou_type="bbox")
# ---------------------------------------------------------------------------
def _random_bbox_batch(
rng: np.random.Generator,
num_shared_preds: int,
num_shared_gts: int,
num_shared_classes: int,
canvas: int,
) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor]]:
"""Build one random single-image bbox pred/target pair for the differential oracle.
*num_shared_classes* random detections and GTs are drawn from a shared label pool, so a class
can end up with predictions and GTs, only predictions, only GTs, or neither (if unlucky) — the
common case ``_accumulate_bbox_class`` must handle via ``np.ix_`` slicing of the shared
image-wide IoU matrix. On top of that pool, one extra prediction is always given a
class id of its own (``num_shared_classes``) with no matching GT, and one extra GT is always
given a class id of its own (``num_shared_classes + 1``) with no matching prediction — this
guarantees the two empty-class edges (`gt_indices.size == 0`, and a GT-only class) are
exercised on every trial rather than only probabilistically.
Args:
rng: Seeded NumPy random generator, local to the caller (not the global RNG state).
num_shared_preds: Number of predictions drawn from the shared class pool.
num_shared_gts: Number of ground truths drawn from the shared class pool.
num_shared_classes: Size of the shared class-id pool.
canvas: Boxes are drawn with coordinates inside a ``canvas``-square area.
Returns:
A ``(pred, target)`` pair shaped like one ``build_matching_data()`` batch element, with a
random ``iscrowd`` flag on every GT.
Examples:
>>> rng = np.random.default_rng(0)
>>> pred, target = _random_bbox_batch(
... rng, num_shared_preds=3, num_shared_gts=2, num_shared_classes=2, canvas=16
... )
>>> pred["boxes"].shape, pred["scores"].shape, pred["labels"].shape
(torch.Size([4, 4]), torch.Size([4]), torch.Size([4]))
>>> target["boxes"].shape, target["labels"].shape, target["iscrowd"].shape
(torch.Size([3, 4]), torch.Size([3]), torch.Size([3]))
"""
def _random_boxes(count: int) -> np.ndarray:
top_left = rng.uniform(0, canvas / 2, size=(count, 2))
extent = rng.uniform(1, canvas / 2, size=(count, 2))
return np.concatenate([top_left, top_left + extent], axis=1)
pred_only_class = num_shared_classes
gt_only_class = num_shared_classes + 1
pred_boxes = np.concatenate([_random_boxes(num_shared_preds), _random_boxes(1)], axis=0)
pred_labels = np.append(rng.integers(0, num_shared_classes, size=num_shared_preds), pred_only_class)
pred_scores = rng.uniform(0.01, 1.0, size=num_shared_preds + 1)
gt_boxes = np.concatenate([_random_boxes(num_shared_gts), _random_boxes(1)], axis=0)
gt_labels = np.append(rng.integers(0, num_shared_classes, size=num_shared_gts), gt_only_class)
gt_crowd = rng.integers(0, 2, size=num_shared_gts + 1)
pred = {
"boxes": torch.tensor(pred_boxes, dtype=torch.float32),
"scores": torch.tensor(pred_scores, dtype=torch.float32),
"labels": torch.tensor(pred_labels, dtype=torch.int64),
}
target = {
"boxes": torch.tensor(gt_boxes, dtype=torch.float32),
"labels": torch.tensor(gt_labels, dtype=torch.int64),
"iscrowd": torch.tensor(gt_crowd, dtype=torch.int64),
}
return pred, target
def _reference_bbox_matching(
pred: dict[str, torch.Tensor],
target: dict[str, torch.Tensor],
iou_threshold: float,
) -> dict[int, tuple[np.ndarray, np.ndarray, int]]:
"""Independently compute build_matching_data's bbox-path output, per class, as an oracle.
Mirrors the sharing strategy ``_accumulate_bbox_class`` uses (one image-wide ``box_iou``
matrix, sliced per class) so that a per-class slicing bug — e.g. a wrong ``np.ix_`` index —
disagrees with this independent per-class computation instead of silently agreeing with
itself. The actual greedy matching for each class is delegated to
``_reference_greedy_match``, which reimplements the matching contract from spec rather than
reusing any of ``rfdetr.evaluation.matching``.
Args:
pred: One image's predictions with ``boxes``, ``scores``, ``labels``.
target: One image's ground truths with ``boxes``, ``labels``, and optional ``iscrowd``.
iou_threshold: Minimum IoU to count as a positive match.
Returns:
Dict mapping ``class_id`` to ``(matches, ignore, total_gt)``, with ``matches``/``ignore``
in the same per-class, descending-score order ``build_matching_data()`` returns.
Examples:
>>> pred = {
... "boxes": torch.tensor([[0.0, 0.0, 10.0, 10.0]]),
... "scores": torch.tensor([0.9]),
... "labels": torch.tensor([0]),
... }
>>> target = {"boxes": torch.tensor([[0.0, 0.0, 10.0, 10.0]]), "labels": torch.tensor([0])}
>>> reference = _reference_bbox_matching(pred, target, iou_threshold=0.5)
>>> reference[0][0].tolist(), reference[0][2]
([1], 1)
"""
pred_labels_np = pred["labels"].numpy()
gt_labels_np = target["labels"].numpy()
scores_np = pred["scores"].numpy()
gt_crowd = target.get("iscrowd", torch.zeros(len(gt_labels_np), dtype=torch.int64)).bool()
iou_matrix = box_iou(pred["boxes"], target["boxes"])
reference: dict[int, tuple[np.ndarray, np.ndarray, int]] = {}
for class_id in sorted(set(pred_labels_np.tolist()) | set(gt_labels_np.tolist())):
pred_idx = np.flatnonzero(pred_labels_np == class_id)
gt_idx = np.flatnonzero(gt_labels_np == class_id)
if pred_idx.size == 0:
reference[class_id] = (
np.zeros(0, dtype=np.int64),
np.zeros(0, dtype=np.bool_),
int((~gt_crowd[gt_idx]).sum()),
)
continue
if gt_idx.size == 0:
reference[class_id] = (
np.zeros(pred_idx.size, dtype=np.int64),
np.zeros(pred_idx.size, dtype=np.bool_),
0,
)
continue
order = np.argsort(-scores_np[pred_idx], kind="stable")
sub_iou = iou_matrix[np.ix_(pred_idx[order], gt_idx)]
matches, ignore, total_gt = _reference_greedy_match(
list(range(len(order))), sub_iou, gt_crowd[gt_idx], iou_threshold
)
reference[class_id] = (matches, ignore, total_gt)
return reference
def _assert_matching_equals_reference(
result: dict[int, dict[str, np.ndarray | int]],
reference: dict[int, tuple[np.ndarray, np.ndarray, int]],
) -> None:
"""Assert build_matching_data's per-class output equals a (matches, ignore, total_gt) reference.
Both ``build_matching_data()`` and ``_reference_bbox_matching()`` key on ``class_id`` and keep
detections in per-image descending-score order, so comparing arrays directly is valid without
re-sorting either side.
Args:
result: Output of ``build_matching_data()``.
reference: ``class_id -> (matches, ignore, total_gt)`` from ``_reference_bbox_matching()``.
Examples:
>>> result = {0: {"matches": np.array([1]), "ignore": np.array([False]), "total_gt": 1}}
>>> reference = {0: (np.array([1]), np.array([False]), 1)}
>>> _assert_matching_equals_reference(result, reference)
"""
assert set(result) == set(reference)
for class_id, (ref_matches, ref_ignore, ref_total_gt) in reference.items():
np.testing.assert_array_equal(result[class_id]["matches"], ref_matches)
np.testing.assert_array_equal(result[class_id]["ignore"], ref_ignore)
assert result[class_id]["total_gt"] == ref_total_gt
class TestBuildMatchingDataBboxDifferential:
"""Randomized differential-oracle coverage for build_matching_data(iou_type="bbox").
``test_greedy_loop_does_not_sync_a_tensor_per_detection`` (see ``TestMatchSingleClassSegm``) is
this suite's only other randomized differential oracle, and it exercises the segm path only.
The bbox path shares one image-wide ``box_iou`` matrix across classes and slices it per class
via ``np.ix_`` (``_accumulate_bbox_class``); every other bbox-path test in
``TestBuildMatchingData`` hand-writes 2-3 detections, none of them drives this class-slicing
with more than one populated class at a time.
"""
@pytest.mark.parametrize("seed", [pytest.param(seed, id=f"seed-{seed}") for seed in range(50)])
def test_matches_independent_reference_across_random_batches(self, seed: int) -> None:
"""build_matching_data(iou_type="bbox") agrees with an independent per-class oracle.
Each of the 50 independently-seeded trials builds one random single-image batch with several shared classes (so
the ``np.ix_`` per-class slice of the one image-wide ``box_iou`` matrix is exercised for more than one class per
trial), a random ``iscrowd`` flag on every GT, one prediction-only class, and one GT-only class (see
``_random_bbox_batch``). Many small, cheap trials are used instead of one large trial: each trial covers only a
handful of classes/detections, so the seed sweep covers far more of the class/crowd/empty combinations across 50
trials than a single larger draw would, for a comparable total detection count.
"""
rng = np.random.default_rng(seed)
pred, target = _random_bbox_batch(rng, num_shared_preds=8, num_shared_gts=8, num_shared_classes=4, canvas=32)
result = build_matching_data([pred], [target], iou_type="bbox")
reference = _reference_bbox_matching(pred, target, iou_threshold=0.5)
_assert_matching_equals_reference(result, reference)
@pytest.mark.parametrize("crowd_flag_scale", [pytest.param(1, id="with-crowd"), pytest.param(0, id="crowd-free")])
@pytest.mark.parametrize("seed", [pytest.param(seed, id=f"seed-{seed}") for seed in range(10)])
def test_single_class_image_matches_independent_reference(self, seed: int, crowd_flag_scale: int) -> None:
"""An image whose detections and GTs are all one class agrees with the same independent oracle.
``_random_bbox_batch`` always adds a prediction-only class and a GT-only class, so the trials above never
produce a single-class image. Relabelling every detection and GT to one class does, and that image takes the
path which skips per-class selection altogether — the one class owns every row and column of the image-wide
``box_iou`` matrix, so there is nothing to select and only the row order changes. Scaling ``iscrowd`` to zero
pairs that with the crowd-free matching loop, which together are what a single-class evaluation actually runs.
"""
rng = np.random.default_rng(seed)
pred, target = _random_bbox_batch(rng, num_shared_preds=8, num_shared_gts=6, num_shared_classes=3, canvas=32)
pred["labels"] = torch.zeros_like(pred["labels"])
target["labels"] = torch.zeros_like(target["labels"])
target["iscrowd"] = target["iscrowd"] * crowd_flag_scale
result = build_matching_data([pred], [target], iou_type="bbox")
reference = _reference_bbox_matching(pred, target, iou_threshold=0.5)
_assert_matching_equals_reference(result, reference)
# ---------------------------------------------------------------------------
# Helper shared by TestMergeMatchingData and TestDistributedMergeMatchingData
# (used by multiple classes, so module-level rather than a staticmethod)