Add ssim test (#314)

Co-authored-by: kevin314 <kevin.lin.cs1@gmail.com>

.github/workflows/pr-test.yml

@@ -32,6 +32,8 @@ jobs:
             encoder-test:
               - 'fastvideo/v1/models/encoders/**'
               - 'fastvideo/v1/models/loaders/**'
+              - 'fastvideo/v1/tests/test_clip_encoder.py'
+              - 'fastvideo/v1/tests/test_llama_encoder.py'
 
   encoder-test:
     needs: change-filter
@@ -71,18 +73,56 @@ jobs:
           --gpu-type "NVIDIA A40"
           --gpu-count 1
           --volume-size 100
-          --test-command "./env_setup.sh &&
-          pip install vllm &&
-          pip install pytest &&
-          pytest ./fastvideo/v1/tests/ --ignore=./fastvideo/v1/tests/old_tests/ -s"
+          --test-command "pip install -e .[test] &&
+          pip install flash-attn==2.7.0.post2 --no-build-isolation &&
+          pytest ./fastvideo/v1/tests/test_clip_encoder.py ./fastvideo/v1/tests/test_llama_encoder.py -s"
+
+  ssim-test:
+    if: ${{ github.event.pull_request.draft == false }}
+    runs-on: ubuntu-latest
+    environment: runpod-runners
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up SSH key
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.RUNPOD_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub
+
+      - name: Install dependencies
+        run: pip install requests
+
+      - name: Run tests on RunPod
+        env:
+          JOB_ID: "ssim-test"
+          RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
+          GITHUB_SHA: ${{ github.sha }}
+          GITHUB_REF: ${{ github.ref }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+        timeout-minutes: 30
+        run: >-
+          python .github/scripts/runpod_api.py 
+          --gpu-type "NVIDIA A40"
+          --gpu-count 2
+          --disk-size 100
+          --volume-size 100
+          --test-command "pip install -e .[test] &&
+          pip install flash-attn==2.7.0.post2 --no-build-isolation &&
+          python scripts/huggingface/download_hf.py --repo_id=FastVideo/FastHunyuan-diffusers --local_dir=data/FastHunyuan-diffusers --repo_type=model &&
+          pytest ./fastvideo/v1/tests/ssim -vs"
 
   runpod-cleanup:
-    needs: [encoder-test] # Add other jobs to this list as you create them
-    if: >-
-      always() && 
-      ${{ github.event.pull_request.draft == false && 
-          (contains(needs.*.result, 'success') || 
-           contains(needs.*.result, 'failure')) }}
+    needs: [encoder-test, ssim-test] # Add other jobs to this list as you create them
+    if: ${{ always() && github.event.pull_request.draft == false && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure'))}}
     runs-on: ubuntu-latest
     environment: runpod-runners
     steps:
@@ -99,7 +139,7 @@ jobs:
 
       - name: Cleanup all RunPod instances
         env:
-          JOB_IDS: '["encoder-test"]' # JSON array of job IDs
+          JOB_IDS: '["encoder-test", "ssim-test"]' # JSON array of job IDs
           RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
           GITHUB_RUN_ID: ${{ github.run_id }}
         run: python .github/scripts/runpod_cleanup.py