139 lines
6.0 KiB
Python
139 lines
6.0 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
|
|
# This source code is licensed under the license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
import numpy as np
|
|
|
|
from typing import Iterable, Mapping, Tuple, Union
|
|
|
|
|
|
def compute_tapvid_metrics(
|
|
query_points: np.ndarray,
|
|
gt_occluded: np.ndarray,
|
|
gt_tracks: np.ndarray,
|
|
pred_occluded: np.ndarray,
|
|
pred_tracks: np.ndarray,
|
|
query_mode: str,
|
|
) -> Mapping[str, np.ndarray]:
|
|
"""Computes TAP-Vid metrics (Jaccard, Pts. Within Thresh, Occ. Acc.)
|
|
See the TAP-Vid paper for details on the metric computation. All inputs are
|
|
given in raster coordinates. The first three arguments should be the direct
|
|
outputs of the reader: the 'query_points', 'occluded', and 'target_points'.
|
|
The paper metrics assume these are scaled relative to 256x256 images.
|
|
pred_occluded and pred_tracks are your algorithm's predictions.
|
|
This function takes a batch of inputs, and computes metrics separately for
|
|
each video. The metrics for the full benchmark are a simple mean of the
|
|
metrics across the full set of videos. These numbers are between 0 and 1,
|
|
but the paper multiplies them by 100 to ease reading.
|
|
Args:
|
|
query_points: The query points, an in the format [t, y, x]. Its size is
|
|
[b, n, 3], where b is the batch size and n is the number of queries
|
|
gt_occluded: A boolean array of shape [b, n, t], where t is the number
|
|
of frames. True indicates that the point is occluded.
|
|
gt_tracks: The target points, of shape [b, n, t, 2]. Each point is
|
|
in the format [x, y]
|
|
pred_occluded: A boolean array of predicted occlusions, in the same
|
|
format as gt_occluded.
|
|
pred_tracks: An array of track predictions from your algorithm, in the
|
|
same format as gt_tracks.
|
|
query_mode: Either 'first' or 'strided', depending on how queries are
|
|
sampled. If 'first', we assume the prior knowledge that all points
|
|
before the query point are occluded, and these are removed from the
|
|
evaluation.
|
|
Returns:
|
|
A dict with the following keys:
|
|
occlusion_accuracy: Accuracy at predicting occlusion.
|
|
pts_within_{x} for x in [1, 2, 4, 8, 16]: Fraction of points
|
|
predicted to be within the given pixel threshold, ignoring occlusion
|
|
prediction.
|
|
jaccard_{x} for x in [1, 2, 4, 8, 16]: Jaccard metric for the given
|
|
threshold
|
|
average_pts_within_thresh: average across pts_within_{x}
|
|
average_jaccard: average across jaccard_{x}
|
|
"""
|
|
|
|
metrics = {}
|
|
# Fixed bug is described in:
|
|
# https://github.com/facebookresearch/co-tracker/issues/20
|
|
eye = np.eye(gt_tracks.shape[2], dtype=np.int32)
|
|
|
|
if query_mode == "first":
|
|
# evaluate frames after the query frame
|
|
query_frame_to_eval_frames = np.cumsum(eye, axis=1) - eye
|
|
elif query_mode == "strided":
|
|
# evaluate all frames except the query frame
|
|
query_frame_to_eval_frames = 1 - eye
|
|
else:
|
|
raise ValueError("Unknown query mode " + query_mode)
|
|
|
|
query_frame = query_points[..., 0]
|
|
query_frame = np.round(query_frame).astype(np.int32)
|
|
evaluation_points = query_frame_to_eval_frames[query_frame] > 0
|
|
|
|
# Occlusion accuracy is simply how often the predicted occlusion equals the
|
|
# ground truth.
|
|
occ_acc = np.sum(
|
|
np.equal(pred_occluded, gt_occluded) & evaluation_points,
|
|
axis=(1, 2),
|
|
) / np.sum(evaluation_points)
|
|
metrics["occlusion_accuracy"] = occ_acc
|
|
|
|
# Next, convert the predictions and ground truth positions into pixel
|
|
# coordinates.
|
|
visible = np.logical_not(gt_occluded)
|
|
pred_visible = np.logical_not(pred_occluded)
|
|
all_frac_within = []
|
|
all_jaccard = []
|
|
for thresh in [1, 2, 4, 8, 16]:
|
|
# True positives are points that are within the threshold and where both
|
|
# the prediction and the ground truth are listed as visible.
|
|
within_dist = np.sum(
|
|
np.square(pred_tracks - gt_tracks),
|
|
axis=-1,
|
|
) < np.square(thresh)
|
|
is_correct = np.logical_and(within_dist, visible)
|
|
|
|
# Compute the frac_within_threshold, which is the fraction of points
|
|
# within the threshold among points that are visible in the ground truth,
|
|
# ignoring whether they're predicted to be visible.
|
|
count_correct = np.sum(
|
|
is_correct & evaluation_points,
|
|
axis=(1, 2),
|
|
)
|
|
count_visible_points = np.sum(visible & evaluation_points, axis=(1, 2))
|
|
frac_correct = count_correct / count_visible_points
|
|
metrics["pts_within_" + str(thresh)] = frac_correct
|
|
all_frac_within.append(frac_correct)
|
|
|
|
true_positives = np.sum(
|
|
is_correct & pred_visible & evaluation_points, axis=(1, 2)
|
|
)
|
|
|
|
# The denominator of the jaccard metric is the true positives plus
|
|
# false positives plus false negatives. However, note that true positives
|
|
# plus false negatives is simply the number of points in the ground truth
|
|
# which is easier to compute than trying to compute all three quantities.
|
|
# Thus we just add the number of points in the ground truth to the number
|
|
# of false positives.
|
|
#
|
|
# False positives are simply points that are predicted to be visible,
|
|
# but the ground truth is not visible or too far from the prediction.
|
|
gt_positives = np.sum(visible & evaluation_points, axis=(1, 2))
|
|
false_positives = (~visible) & pred_visible
|
|
false_positives = false_positives | ((~within_dist) & pred_visible)
|
|
false_positives = np.sum(false_positives & evaluation_points, axis=(1, 2))
|
|
jaccard = true_positives / (gt_positives + false_positives)
|
|
metrics["jaccard_" + str(thresh)] = jaccard
|
|
all_jaccard.append(jaccard)
|
|
metrics["average_jaccard"] = np.mean(
|
|
np.stack(all_jaccard, axis=1),
|
|
axis=1,
|
|
)
|
|
metrics["average_pts_within_thresh"] = np.mean(
|
|
np.stack(all_frac_within, axis=1),
|
|
axis=1,
|
|
)
|
|
return metrics
|