diff --git a/.buildkite/scripts/pr_test.sh b/.buildkite/scripts/pr_test.sh index 543cb8d54..cadf79be6 100755 --- a/.buildkite/scripts/pr_test.sh +++ b/.buildkite/scripts/pr_test.sh @@ -124,8 +124,43 @@ case "$TEST_TYPE" in MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_lora_extraction_tests" ;; "performance") - log "Running performance tests..." + # log "Running performance tests..." + # MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_performance_tests" + # ;; + log "Running performance tests on Modal..." MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_performance_tests" + + eval "$MODAL_COMMAND" + TEST_EXIT_CODE=$? + + # 1. Define variables + SHORT_SHA=${BUILDKITE_COMMIT:0:7} + REMOTE_DIR="perf_reports/" + LOCAL_DIR="downloaded_reports" + + log "Downloading reports directory from Modal Volume..." + + # 2. Download the whole folder to a temporary local directory + # modal volume get + mkdir -p "$LOCAL_DIR" + modal volume get hf-model-weights "$REMOTE_DIR" "$LOCAL_DIR" + + # 3. Find the specific file for THIS commit + # We look inside the downloaded folder for a file containing our SHORT_SHA + TARGET_FILE=$(ls "$LOCAL_DIR" | grep "$SHORT_SHA" | head -n 1) + + if [ -n "$TARGET_FILE" ]; then + LOCAL_PATH="$LOCAL_DIR/$TARGET_FILE" + log "Found dashboard: $TARGET_FILE. Uploading to Buildkite..." + + buildkite-agent artifact upload "$LOCAL_PATH" + buildkite-agent annotate --style info --context "perf" < "$LOCAL_PATH" + else + log "Warning: Could not find a dashboard file matching $SHORT_SHA in $REMOTE_DIR" + fi + + # Clean up the temporary folder + rm -rf "$LOCAL_DIR" ;; "api_server") log "Running API server integration tests..." diff --git a/fastvideo/tests/performance/dashboard.py b/fastvideo/tests/performance/dashboard.py index 8c4b38c87..eea2b26fa 100644 --- a/fastvideo/tests/performance/dashboard.py +++ b/fastvideo/tests/performance/dashboard.py @@ -197,7 +197,7 @@ def upload_artifact(path: str): ["buildkite-agent", "artifact", "upload", path], check=True, ) - + # ----------------------------- # 7. Main # ----------------------------- @@ -215,15 +215,39 @@ def main(): figs = build_plots(df) html = render_html(figs) - output_file = f"dashboard_{datetime.now().strftime('%Y%m%d_%H%M%S')}.html" + # Get the commit SHA from environment (passed via MODAL_ENV in your shell script) + commit_sha = os.environ.get("BUILDKITE_COMMIT", "unknown")[:7] + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + + report_dir = "/root/data/perf_reports" + os.makedirs(report_dir, exist_ok=True) + + # Include the SHA in the filename! + # Example: dashboard_a1b2c3d_20240426_032528.html + filename = f"dashboard_{commit_sha}_{timestamp}.html" + output_file = os.path.join(report_dir, filename) + with open(output_file, "w") as f: f.write(html) - annotate_buildkite(html) print(f"Dashboard generated: {output_file}") - upload_artifact(output_file) - print(f"Dashboard uploaded as Buildkite artifact: {output_file}") + # report_dir = "/root/data/perf_reports" + + # # 2. Create the directory (if it doesn't exist) + # os.makedirs(report_dir, exist_ok=True) + + # # 3. Define the full file path + # output_file = os.path.join(report_dir, f"dashboard_{datetime.now().strftime('%Y%m%d_%H%M%S')}.html") + + # with open(output_file, "w") as f: + # f.write(html) + + # annotate_buildkite(html) + # print(f"Dashboard generated: {output_file}") + + # upload_artifact(output_file) + # print(f"Dashboard uploaded as Buildkite artifact: {output_file}") if __name__ == "__main__": main() \ No newline at end of file