244 lines
7.2 KiB
Bash
244 lines
7.2 KiB
Bash
#!/usr/bin/env bash
|
|
# YAPF formatter, adapted from fastvideo.
|
|
#
|
|
# Usage:
|
|
# # Do work and commit your work.
|
|
|
|
# # Format files that differ from origin/main.
|
|
# bash format.sh
|
|
|
|
# # Commit changed files with message 'Run yapf and ruff'
|
|
#
|
|
#
|
|
# This script formats all changed files from the last mergebase.
|
|
# You are encouraged to run this locally before pushing changes for review.
|
|
|
|
# Cause the script to exit if a single command fails
|
|
set -eo pipefail
|
|
|
|
# this stops git rev-parse from failing if we run this from the .git directory
|
|
builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
|
|
ROOT="$(git rev-parse --show-toplevel)"
|
|
builtin cd "$ROOT" || exit 1
|
|
|
|
check_command() {
|
|
if ! command -v "$1" &> /dev/null; then
|
|
echo "❓❓$1 is not installed, please run \`bash env_setup.sh\`"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
check_command yapf
|
|
check_command ruff
|
|
check_command codespell
|
|
check_command isort
|
|
|
|
YAPF_VERSION=$(yapf --version | awk '{print $2}')
|
|
RUFF_VERSION=$(ruff --version | awk '{print $2}')
|
|
CODESPELL_VERSION=$(codespell --version)
|
|
ISORT_VERSION=$(isort --vn)
|
|
SPHINX_LINT_VERSION=$(sphinx-lint --version | awk '{print $2}')
|
|
|
|
|
|
# # params: tool name, tool version, required version
|
|
tool_version_check() {
|
|
expected=$(grep "$1" requirements-lint.txt | cut -d'=' -f3)
|
|
if [[ "$2" != "$expected" ]]; then
|
|
echo "❓❓Wrong $1 version installed: $expected is required, not $2."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
tool_version_check "yapf" "$YAPF_VERSION"
|
|
tool_version_check "ruff" "$RUFF_VERSION"
|
|
tool_version_check "isort" "$ISORT_VERSION"
|
|
tool_version_check "codespell" "$CODESPELL_VERSION"
|
|
tool_version_check "sphinx-lint" "$SPHINX_LINT_VERSION"
|
|
|
|
YAPF_FLAGS=(
|
|
'--recursive'
|
|
'--parallel'
|
|
)
|
|
|
|
YAPF_EXCLUDES=(
|
|
'--exclude' 'data/**'
|
|
)
|
|
|
|
# Format specified files
|
|
format() {
|
|
yapf --in-place "${YAPF_FLAGS[@]}" "$@"
|
|
}
|
|
|
|
# Format files that differ from main branch. Ignores dirs that are not slated
|
|
# for autoformat yet.
|
|
format_changed() {
|
|
# The `if` guard ensures that the list of filenames is not empty, which
|
|
# could cause yapf to receive 0 positional arguments, making it hang
|
|
# waiting for STDIN.
|
|
#
|
|
# `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
|
|
# exist on both branches.
|
|
MERGEBASE="$(git merge-base origin/main HEAD)"
|
|
|
|
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
|
|
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs -P 5 \
|
|
yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
|
|
fi
|
|
|
|
}
|
|
|
|
# Format all files
|
|
format_all() {
|
|
yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" .
|
|
}
|
|
|
|
## This flag formats individual files. --files *must* be the first command line
|
|
## arg to use this option.
|
|
if [[ "$1" == '--files' ]]; then
|
|
format "${@:2}"
|
|
# If `--all` is passed, then any further arguments are ignored and the
|
|
# entire python directory is formatted.
|
|
elif [[ "$1" == '--all' ]]; then
|
|
format_all
|
|
else
|
|
# Format only the files that changed in last commit.
|
|
format_changed
|
|
fi
|
|
echo 'FastVideo yapf: Done'
|
|
|
|
|
|
# If git diff returns a file that is in the skip list, the file may be checked anyway:
|
|
# https://github.com/codespell-project/codespell/issues/1915
|
|
# Avoiding the "./" prefix and using "/**" globs for directories appears to solve the problem
|
|
CODESPELL_EXCLUDES=(
|
|
'--skip' 'data/**,
|
|
fastvideo/distill.py,
|
|
fastvideo/models/hunyuan/modules/models.py,
|
|
fastvideo/models/mochi_hf/modeling_mochi.py,
|
|
fastvideo/utils/env_utils.py,
|
|
./csrc/sliding_tile_attention/tk/**'
|
|
|
|
)
|
|
|
|
# check spelling of specified files
|
|
spell_check() {
|
|
codespell "$@"
|
|
}
|
|
|
|
spell_check_all(){
|
|
codespell --toml pyproject.toml
|
|
}
|
|
|
|
# Spelling check of files that differ from main branch.
|
|
spell_check_changed() {
|
|
# The `if` guard ensures that the list of filenames is not empty, which
|
|
# could cause ruff to receive 0 positional arguments, making it hang
|
|
# waiting for STDIN.
|
|
#
|
|
# `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
|
|
# exist on both branches.
|
|
MERGEBASE="$(git merge-base origin/main HEAD)"
|
|
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
|
|
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
|
|
codespell "${CODESPELL_EXCLUDES[@]}"
|
|
fi
|
|
}
|
|
|
|
# Run Codespell
|
|
## This flag runs spell check of individual files. --files *must* be the first command line
|
|
# arg to use this option.
|
|
if [[ "$1" == '--files' ]]; then
|
|
spell_check "${@:2}"
|
|
# If `--all` is passed, then any further arguments are ignored and the
|
|
# entire python directory is linted.
|
|
elif [[ "$1" == '--all' ]]; then
|
|
spell_check_all
|
|
else
|
|
# Check spelling only of the files that changed in last commit.
|
|
spell_check_changed
|
|
fi
|
|
echo 'FastVideo codespell: Done'
|
|
|
|
|
|
# Lint specified files
|
|
lint() {
|
|
ruff check "$@"
|
|
}
|
|
|
|
# Lint files that differ from main branch. Ignores dirs that are not slated
|
|
# for autolint yet.
|
|
lint_changed() {
|
|
# The `if` guard ensures that the list of filenames is not empty, which
|
|
# could cause ruff to receive 0 positional arguments, making it hang
|
|
# waiting for STDIN.
|
|
#
|
|
# `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
|
|
# exist on both branches.
|
|
MERGEBASE="$(git merge-base origin/main HEAD)"
|
|
|
|
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
|
|
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
|
|
ruff check
|
|
fi
|
|
|
|
}
|
|
|
|
# Run Ruff
|
|
### This flag lints individual files. --files *must* be the first command line
|
|
### arg to use this option.
|
|
if [[ "$1" == '--files' ]]; then
|
|
lint "${@:2}"
|
|
# If `--all` is passed, then any further arguments are ignored and the
|
|
# entire python directory is linted.
|
|
elif [[ "$1" == '--all' ]]; then
|
|
lint fastvideo scripts
|
|
else
|
|
# Format only the files that changed in last commit.
|
|
lint_changed
|
|
fi
|
|
echo 'FastVideo ruff: Done'
|
|
|
|
ISORT_EXCLUDES=(
|
|
'--skip' './csrc/sliding_tile_attention/tk'
|
|
)
|
|
|
|
# check spelling of specified files
|
|
isort_check() {
|
|
isort "$@"
|
|
}
|
|
|
|
isort_check_all(){
|
|
isort "${ISORT_EXCLUDES[@]}" .
|
|
}
|
|
|
|
# Spelling check of files that differ from main branch.
|
|
isort_check_changed() {
|
|
# The `if` guard ensures that the list of filenames is not empty, which
|
|
# could cause ruff to receive 0 positional arguments, making it hang
|
|
# waiting for STDIN.
|
|
#
|
|
# `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
|
|
# exist on both branches.
|
|
MERGEBASE="$(git merge-base origin/main HEAD)"
|
|
|
|
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
|
|
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
|
|
isort
|
|
fi
|
|
}
|
|
|
|
# Run Isort
|
|
# This flag runs spell check of individual files. --files *must* be the first command line
|
|
# arg to use this option.
|
|
if [[ "$1" == '--files' ]]; then
|
|
isort_check "${@:2}"
|
|
# If `--all` is passed, then any further arguments are ignored and the
|
|
# entire python directory is linted.
|
|
elif [[ "$1" == '--all' ]]; then
|
|
isort_check_all
|
|
else
|
|
# Check spelling only of the files that changed in last commit.
|
|
isort_check_changed
|
|
fi
|
|
echo 'FastVideo isort: Done'
|