239 lines
8.7 KiB
YAML
239 lines
8.7 KiB
YAML
name: LanPaintBench LPIPS (ImageNet subset)
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
base_ref:
|
|
description: "Base ref to benchmark against (commit SHA or branch). Leave empty to use repo default branch."
|
|
required: false
|
|
default: ""
|
|
subset:
|
|
description: "Number of ImageNet images to benchmark"
|
|
required: true
|
|
default: "1"
|
|
sample_steps:
|
|
description: "Outer diffusion steps (guided_diffusion_e2e)"
|
|
required: true
|
|
default: "20"
|
|
n_steps:
|
|
description: "LanPaint inner steps per outer step"
|
|
required: true
|
|
default: "5"
|
|
seed:
|
|
description: "Random seed"
|
|
required: true
|
|
default: "42"
|
|
shuffle_buffer:
|
|
description: "HF streaming shuffle buffer"
|
|
required: true
|
|
default: "1024"
|
|
semantic_threshold:
|
|
description: "Enable semantic early-stop if > 0 (head_on run)"
|
|
required: true
|
|
default: "0.01"
|
|
semantic_patience:
|
|
description: "Semantic early-stop patience (head_on run)"
|
|
required: true
|
|
default: "2"
|
|
min_seconds:
|
|
description: "Fail if a benchmark runtime is below this (0 to disable)"
|
|
required: true
|
|
default: "0"
|
|
max_case_mean_delta:
|
|
description: "Fail if mean per-case metric delta exceeds this (head_on - head_off)"
|
|
required: true
|
|
default: "0.01"
|
|
max_case_p95_delta:
|
|
description: "Fail if P95 per-case metric delta exceeds this (head_on - head_off)"
|
|
required: true
|
|
default: "0.02"
|
|
max_case_max_delta:
|
|
description: "Fail if max per-case metric delta exceeds this (head_on - head_off)"
|
|
required: true
|
|
default: "0.05"
|
|
gallery_topk:
|
|
description: "How many worst-case examples to render in the gallery"
|
|
required: true
|
|
default: "8"
|
|
|
|
jobs:
|
|
benchmark:
|
|
name: Compare Base vs Head
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 360
|
|
env:
|
|
HF_HOME: ~/.cache/huggingface
|
|
BASE_REF: ${{ inputs.base_ref || github.event.repository.default_branch }}
|
|
OMP_NUM_THREADS: "1"
|
|
MKL_NUM_THREADS: "1"
|
|
LANPAINTBENCH_REF: "ea5be5d"
|
|
CHECKPOINT_URL: "https://openaipublic.blob.core.windows.net/diffusion/jul-2021/256x256_diffusion_uncond.pt"
|
|
steps:
|
|
- name: Checkout Head (Current PR)
|
|
uses: actions/checkout@v4
|
|
with:
|
|
path: head
|
|
fetch-depth: 1
|
|
|
|
- name: Checkout Base (Baseline)
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ env.BASE_REF }}
|
|
path: base
|
|
fetch-depth: 1
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
cache: 'pip'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
|
|
pip install lpips datasets numpy pillow blobfile
|
|
# We don't need to install LanPaint as a package because we use sys.path in the script
|
|
|
|
- name: Cache HF + torch + checkpoints
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/huggingface
|
|
~/.cache/torch
|
|
~/.cache/lanpaintbench
|
|
key: ${{ runner.os }}-lpips-imagenet-${{ hashFiles('head/tests/benchmark_lpips.py', 'head/tests/compare_benchmarks.py', 'head/tests/render_benchmark_gallery.py') }}
|
|
|
|
- name: Clone LanPaintBench
|
|
run: |
|
|
git clone https://github.com/scraed/LanPaintBench.git bench
|
|
cd bench
|
|
git checkout "$LANPAINTBENCH_REF"
|
|
|
|
- name: Download ImageNet-256 checkpoint
|
|
run: |
|
|
CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
|
|
mkdir -p ~/.cache/lanpaintbench/checkpoints
|
|
if [ ! -f "$CHECKPOINT_PATH" ]; then
|
|
curl -L "$CHECKPOINT_URL" -o "$CHECKPOINT_PATH"
|
|
fi
|
|
ls -lh "$CHECKPOINT_PATH"
|
|
|
|
- name: Run Baseline Benchmark
|
|
run: |
|
|
CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
|
|
# Use the benchmark script from the HEAD branch to ensure test consistency
|
|
python head/tests/benchmark_lpips.py \
|
|
--mode guided_diffusion_e2e \
|
|
--src base/src \
|
|
--lanpaintbench bench \
|
|
--checkpoint "$CHECKPOINT_PATH" \
|
|
--output baseline.json \
|
|
--dataset benjamin-paine/imagenet-1k-256x256 \
|
|
--split train \
|
|
--subset "${{ inputs.subset }}" \
|
|
--seed "${{ inputs.seed }}" \
|
|
--shuffle-buffer "${{ inputs.shuffle_buffer }}" \
|
|
--no-shuffle \
|
|
--mask-type box \
|
|
--n-steps "${{ inputs.n_steps }}" \
|
|
--sample-steps "${{ inputs.sample_steps }}" \
|
|
--device cpu \
|
|
--min-seconds "${{ inputs.min_seconds }}" \
|
|
--semantic-threshold 0.0
|
|
|
|
- name: Run Head Benchmark (early-stop OFF)
|
|
run: |
|
|
CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
|
|
python head/tests/benchmark_lpips.py \
|
|
--mode guided_diffusion_e2e \
|
|
--src head/src \
|
|
--lanpaintbench bench \
|
|
--checkpoint "$CHECKPOINT_PATH" \
|
|
--output head_off.json \
|
|
--dataset benjamin-paine/imagenet-1k-256x256 \
|
|
--split train \
|
|
--subset "${{ inputs.subset }}" \
|
|
--seed "${{ inputs.seed }}" \
|
|
--shuffle-buffer "${{ inputs.shuffle_buffer }}" \
|
|
--no-shuffle \
|
|
--mask-type box \
|
|
--n-steps "${{ inputs.n_steps }}" \
|
|
--sample-steps "${{ inputs.sample_steps }}" \
|
|
--device cpu \
|
|
--min-seconds "${{ inputs.min_seconds }}" \
|
|
--semantic-threshold 0.0 \
|
|
--save-images-dir head_off_images
|
|
|
|
- name: Run Head Benchmark (early-stop ON)
|
|
run: |
|
|
CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
|
|
python head/tests/benchmark_lpips.py \
|
|
--mode guided_diffusion_e2e \
|
|
--src head/src \
|
|
--lanpaintbench bench \
|
|
--checkpoint "$CHECKPOINT_PATH" \
|
|
--output head_on.json \
|
|
--dataset benjamin-paine/imagenet-1k-256x256 \
|
|
--split train \
|
|
--subset "${{ inputs.subset }}" \
|
|
--seed "${{ inputs.seed }}" \
|
|
--shuffle-buffer "${{ inputs.shuffle_buffer }}" \
|
|
--no-shuffle \
|
|
--mask-type box \
|
|
--n-steps "${{ inputs.n_steps }}" \
|
|
--sample-steps "${{ inputs.sample_steps }}" \
|
|
--device cpu \
|
|
--min-seconds "${{ inputs.min_seconds }}" \
|
|
--semantic-threshold "${{ inputs.semantic_threshold }}" \
|
|
--semantic-patience "${{ inputs.semantic_patience }}" \
|
|
--save-images-dir head_on_images
|
|
|
|
- name: Compare baseline vs head_off
|
|
continue-on-error: true
|
|
run: |
|
|
python head/tests/compare_benchmarks.py --baseline baseline.json --current head_off.json --max-rel-diff 0.01 --max-abs-diff 0.0
|
|
|
|
- name: Compare head_off vs head_on
|
|
run: |
|
|
python head/tests/compare_benchmarks.py \
|
|
--baseline head_off.json \
|
|
--current head_on.json \
|
|
--max-rel-diff 0.01 \
|
|
--max-abs-diff 0.0 \
|
|
--case-metric lpips_inpaint_ctx \
|
|
--max-case-mean-delta "${{ inputs.max_case_mean_delta }}" \
|
|
--max-case-p95-delta "${{ inputs.max_case_p95_delta }}" \
|
|
--max-case-max-delta "${{ inputs.max_case_max_delta }}"
|
|
|
|
- name: Compare baseline vs head_on
|
|
continue-on-error: true
|
|
run: |
|
|
python head/tests/compare_benchmarks.py --baseline baseline.json --current head_on.json --max-rel-diff 0.01 --max-abs-diff 0.0
|
|
|
|
- name: Render worst-case gallery (head_off vs head_on)
|
|
if: ${{ always() }}
|
|
run: |
|
|
python head/tests/render_benchmark_gallery.py \
|
|
--baseline-json head_off.json \
|
|
--current-json head_on.json \
|
|
--baseline-images head_off_images \
|
|
--current-images head_on_images \
|
|
--out-dir gallery \
|
|
--metric lpips_inpaint_ctx \
|
|
--top-k "${{ inputs.gallery_topk }}"
|
|
|
|
- name: Upload benchmark JSON
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: lpips-benchmark-artifacts
|
|
path: |
|
|
baseline.json
|
|
head_off.json
|
|
head_on.json
|
|
head_off_images
|
|
head_on_images
|
|
gallery
|
|
if-no-files-found: warn
|