diff --git a/.github/scripts/runpod_api.py b/.github/scripts/runpod_api.py index 9ebdd0b04..eb77e6599 100644 --- a/.github/scripts/runpod_api.py +++ b/.github/scripts/runpod_api.py @@ -28,7 +28,7 @@ def parse_arguments(): parser.add_argument( '--image', type=str, - default='runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04', + required=True, help='Docker image to use') return parser.parse_args() @@ -46,6 +46,16 @@ HEADERS = { def create_pod(): """Create a RunPod instance""" + # Ensure image name is lowercase (Docker requirement) + image_name = args.image.lower() + print(f"Using specified image: {image_name}") + + docker_start_cmd = [ + "bash", + "-c", + "apt update;DEBIAN_FRONTEND=noninteractive apt-get install openssh-server -y;mkdir -p ~/.ssh;cd $_;chmod 700 ~/.ssh;echo \"$PUBLIC_KEY\" >> authorized_keys;chmod 700 authorized_keys;service ssh start;sleep infinity" + ] + print(f"Creating RunPod instance with GPU: {args.gpu_type}...") payload = { "name": f"fastvideo-{JOB_ID}-{RUN_ID}", @@ -53,8 +63,9 @@ def create_pod(): "volumeInGb": args.volume_size, "gpuTypeIds": [args.gpu_type], "gpuCount": args.gpu_count, - "imageName": args.image, - "allowedCudaVersions": ["12.4"] + "imageName": image_name, + "allowedCudaVersions": ["12.4"], + "dockerStartCmd": docker_start_cmd } response = requests.post(PODS_API, headers=HEADERS, json=payload) @@ -91,7 +102,7 @@ def wait_for_pod(pod_id): "Timed out waiting for RunPod to reach RUNNING state") # Wait for ports to be assigned - max_attempts = 6 + max_attempts = 35 attempts = 0 while attempts < max_attempts: response = requests.get(f"{PODS_API}/{pod_id}", headers=HEADERS) @@ -108,7 +119,7 @@ def wait_for_pod(pod_id): print( f"Waiting for SSH port and public IP to be available... (attempt {attempts+1}/{max_attempts})" ) - time.sleep(10) + time.sleep(20) attempts += 1 if attempts >= max_attempts: @@ -145,16 +156,15 @@ def execute_command(pod_id): ] subprocess.run(scp_command, check=True) + # For custom image, we can use the pre-configured environment setup_steps = [ - "cd /workspace", - "wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh", - "bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda3", - "source $HOME/miniconda3/bin/activate", - "conda create --name venv python=3.10.0 -y", "conda activate venv", - "mkdir -p /workspace/repo", "tar -xzf /tmp/repo.tar.gz --no-same-owner -C /workspace/", - f"cd /workspace/{repo_name}", args.test_command + f"cd /workspace/{repo_name}", + "source /opt/conda/etc/profile.d/conda.sh", + "conda activate fastvideo-dev", + args.test_command ] + remote_command = " && ".join(setup_steps) ssh_command = [ diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 000000000..52a445393 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,78 @@ +name: Build and Push Docker Image + +on: + workflow_dispatch: # Only manual triggers + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Free up disk space + run: | + # Display initial space + echo "Initial disk space:" + df -h + + # Remove large directories directly + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + sudo rm -rf /usr/share/swift + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/share/rust + sudo rm -rf /usr/local/.ghcup + + # Remove cached files + sudo rm -rf /var/lib/apt/lists/* + sudo rm -rf /var/cache/apt/archives/* + + # Clean Docker + docker system prune -af --volumes + + # Display available space after cleanup + echo "Disk space after cleanup:" + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }}/fastvideo-dev + tags: | + type=raw,value=latest + type=sha,format=short + + - name: Build and push Docker image + id: build-push + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Success message + run: | + echo "✅ Image successfully built and pushed to ghcr.io/${{ github.repository }}/fastvideo-dev:latest" + echo "To run tests with this image, manually trigger the 'Run Tests' workflow." \ No newline at end of file diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index afb450e66..89f178bcb 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -14,6 +14,11 @@ on: - ".github/workflows/pr-test.yml" workflow_dispatch: inputs: + custom_image: + description: "Custom image from this repository (default: fastvideo-dev:latest)" + required: false + default: "fastvideo-dev:latest" + type: string run_encoder_test: description: "Run encoder-test" required: false @@ -35,6 +40,9 @@ on: default: false type: boolean +env: + PYTHONUNBUFFERED: "1" + concurrency: group: pr-test-${{ github.ref }} cancel-in-progress: true @@ -107,9 +115,8 @@ jobs: --gpu-type "NVIDIA A40" --gpu-count 1 --volume-size 100 - --test-command "pip install -e .[test] && - pip install flash-attn==2.7.0.post2 --no-build-isolation && - pytest ./fastvideo/v1/tests/encoders -s" + --image "ghcr.io/${{ github.repository }}/${{ github.event.inputs.custom_image || 'fastvideo-dev:latest' }}" + --test-command "pip install -e . && pytest ./fastvideo/v1/tests/encoders -s" - name: Terminate RunPod Instances if: ${{ always() }} @@ -156,9 +163,8 @@ jobs: --gpu-type "NVIDIA A40" --gpu-count 1 --volume-size 100 - --test-command "pip install -e .[test] && - pip install flash-attn==2.7.0.post2 --no-build-isolation && - pytest ./fastvideo/v1/tests/vaes -s" + --image "ghcr.io/${{ github.repository }}/${{ github.event.inputs.custom_image || 'fastvideo-dev:latest' }}" + --test-command "pip install -e . && pytest ./fastvideo/v1/tests/vaes -s" - name: Terminate RunPod Instances if: ${{ always() }} @@ -205,9 +211,8 @@ jobs: --gpu-type "NVIDIA L40S" --gpu-count 1 --volume-size 100 - --test-command "pip install -e .[test] && - pip install flash-attn==2.7.0.post2 --no-build-isolation && - pytest ./fastvideo/v1/tests/transformers -s" + --image "ghcr.io/${{ github.repository }}/${{ github.event.inputs.custom_image || 'fastvideo-dev:latest' }}" + --test-command "pip install -e . && pytest ./fastvideo/v1/tests/transformers -s" - name: Terminate RunPod Instances if: ${{ always() }} @@ -255,9 +260,8 @@ jobs: --gpu-count 2 --disk-size 200 --volume-size 200 - --test-command "pip install -e .[test] && - pip install flash-attn==2.7.0.post2 --no-build-isolation && - pytest ./fastvideo/v1/tests/ssim -vs" + --image "ghcr.io/${{ github.repository }}/${{ github.event.inputs.custom_image || 'fastvideo-dev:latest' }}" + --test-command "pip install -e . && pytest ./fastvideo/v1/tests/ssim -vs" - name: Terminate RunPod Instances if: ${{ always() }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..6cca00620 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM nvidia/cuda:12.4.1-devel-ubuntu20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + git \ + ca-certificates \ + openssh-server \ + && rm -rf /var/lib/apt/lists/* + +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ + rm Miniconda3-latest-Linux-x86_64.sh + +ENV PATH=/opt/conda/bin:$PATH + +RUN conda create --name fastvideo-dev python=3.10.0 -y + +SHELL ["/bin/bash", "-c"] + +# Copy just the pyproject.toml first to leverage Docker cache +COPY pyproject.toml ./ + +# Create a dummy README to satisfy the installation +RUN echo "# Placeholder" > README.md + +RUN conda run -n fastvideo-dev pip install --no-cache-dir --upgrade pip && \ + conda run -n fastvideo-dev pip install --no-cache-dir .[dev] && \ + conda run -n fastvideo-dev pip install --no-cache-dir flash-attn==2.7.0.post2 --no-build-isolation && \ + conda clean -afy + +COPY . . + +EXPOSE 22 \ No newline at end of file