From 4840690039096dbad3d83f9ca8bd711164b916ce Mon Sep 17 00:00:00 2001 From: Denis Kochetov Date: Wed, 11 Jun 2025 01:49:31 +0300 Subject: [PATCH] updated installation to cuda 12.8 --- _Pre_Builds/_Build_Scripts/auto_build_all.py | 2 + _Pre_Builds/_Build_Scripts/build_config.yaml | 22 ++-- .../_Build_Scripts/build_requirements.txt | 1 + _Pre_Builds/_Build_Scripts/build_utils.py | 119 ++++++++++++++++-- _Pre_Builds/_Build_Scripts/dependencies.txt | 1 + install.py | 15 ++- requirements.txt | 5 + 7 files changed, 149 insertions(+), 16 deletions(-) diff --git a/_Pre_Builds/_Build_Scripts/auto_build_all.py b/_Pre_Builds/_Build_Scripts/auto_build_all.py index f12c923..09e084f 100644 --- a/_Pre_Builds/_Build_Scripts/auto_build_all.py +++ b/_Pre_Builds/_Build_Scripts/auto_build_all.py @@ -13,6 +13,7 @@ from build_utils import ( calculate_runtime, git_folder_parallel, install_remote_packages, + install_platform_packages, build_config, PYTHON_PATH, DEPENDENCIES_FILE_ABS_PATH, @@ -49,6 +50,7 @@ def setup_build_env(): subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", "-r", BUILD_REQUIREMENTS_FILE_ABS_PATH]) install_remote_packages(build_config.build_base_packages) + install_platform_packages() def clone_or_update_read_dependency(dependency_url, dependency_dir): if os.path.exists(dependency_dir): diff --git a/_Pre_Builds/_Build_Scripts/build_config.yaml b/_Pre_Builds/_Build_Scripts/build_config.yaml index 47de545..543b8a8 100644 --- a/_Pre_Builds/_Build_Scripts/build_config.yaml +++ b/_Pre_Builds/_Build_Scripts/build_config.yaml @@ -1,6 +1,6 @@ -repo_id: "MrForExample/Comfy3D_Pre_Builds" -supported_cuda_versions: ["12.4", "12.1", "11.8"] -cuda_version: "12.4" +repo_id: "DenisKochetov/Comfy3D_Pre_Builds" +supported_cuda_versions: ["12.8", "12.6", "12.4", "12.1", "11.8"] +cuda_version: "12.8" # Packages inside this file will be pip downloaded before actual build start build_requirements: "build_requirements.txt" @@ -13,19 +13,27 @@ libs_dir_name: "_Libs" # Folder used to store built wheels wheels_dir_name: "_Build_Wheels" -build_base_packages: ["torch", "torchvision", "xformers", "spconv-${cuda_version}"] +build_base_packages: ["torch", "torchvision", "xformers"] + +# Platform-specific packages +platform_packages: + win: + - "triton-windows<3.4" + linux: + - "triton" + # All packages will be automatically installed at beginning of install.py follwing orders from top to bottom remote_packages: torch: - version: "2.5.1" + version: "2.7.0" url: "https://download.pytorch.org/whl/${cuda_version}" torchvision: - version: "0.20.1" + version: "0.22.0" url: "https://download.pytorch.org/whl/${cuda_version}" xformers: - version: "0.0.28.post3" + version: "0.0.30" url: "https://download.pytorch.org/whl/${cuda_version}" # At the time of writing, torch-scatter is not available for PyTorch 2.5.0 diff --git a/_Pre_Builds/_Build_Scripts/build_requirements.txt b/_Pre_Builds/_Build_Scripts/build_requirements.txt index f58dc0d..6581f0f 100644 --- a/_Pre_Builds/_Build_Scripts/build_requirements.txt +++ b/_Pre_Builds/_Build_Scripts/build_requirements.txt @@ -1,6 +1,7 @@ # build tools cmake ninja +pccm # web tools PyGithub diff --git a/_Pre_Builds/_Build_Scripts/build_utils.py b/_Pre_Builds/_Build_Scripts/build_utils.py index 21e490c..559cb22 100644 --- a/_Pre_Builds/_Build_Scripts/build_utils.py +++ b/_Pre_Builds/_Build_Scripts/build_utils.py @@ -4,6 +4,7 @@ from os.path import dirname import platform import subprocess import time +import glob PYTHON_PATH = sys.executable @@ -40,13 +41,22 @@ def get_pytorch_version(): def get_cuda_version(): # Output: e.g. "cu121" or cu118 - result = subprocess.run(["nvcc", "--version"], text=True, capture_output=True) - if result.returncode == 0: - for cuda_version in build_config.supported_cuda_versions: - if "cuda_" + cuda_version in result.stdout: - return "cu" + cuda_version.replace(".", "") + try: + result = subprocess.run(["nvcc", "--version"], text=True, capture_output=True) + if result.returncode == 0: + for cuda_version in build_config.supported_cuda_versions: + if "cuda_" + cuda_version in result.stdout: + return "cu" + cuda_version.replace(".", "") - raise RuntimeError(f"Please install/reinstall CUDA tookit with any of the following supported version: {build_config.supported_cuda_versions}") + # If nvcc command succeeded but no supported CUDA version found, use default + print(f"Warning: No supported CUDA version detected, using default version: {build_config.cuda_version}") + return "cu" + build_config.cuda_version.replace(".", "") + + except Exception as e: + # If nvcc command failed or any other error occurred, CUDA is not installed + print("CUDA toolkit not found. Please install CUDA 12.8:") + print("https://developer.nvidia.com/cuda-12-8-0-download-archive") + sys.exit(1) OS_TYPE = get_os_type() PYTHON_VERSION = get_python_version() @@ -138,12 +148,76 @@ def git_folder_parallel(repo_id: str, folder: str, recursive: bool=True, root_ou print(f"Couldn't download folder {folder} from repo {repo_id}", e) return False +def is_package_installed(package_name, required_version=None): + """Check if a package is installed with the required version""" + try: + import importlib.metadata + installed_version = importlib.metadata.version(package_name) + + if required_version is None: + return True + + # Handle versions with suffixes like "2.7.1+cu128" + installed_base_version = installed_version.split('+')[0] + required_base_version = required_version.split('+')[0] + + # For PyTorch-related packages, we're more flexible with patch versions + if package_name in ['torch', 'torchvision', 'xformers']: + # Parse version components + try: + installed_parts = [int(x) for x in installed_base_version.split('.')] + required_parts = [int(x) for x in required_base_version.split('.')] + print(f"[DEBUG] {package_name}: version parts - installed={installed_parts}, required={required_parts}") + + # Pad with zeros if needed (e.g., "2.7" vs "2.7.0") + max_len = max(len(installed_parts), len(required_parts)) + installed_parts.extend([0] * (max_len - len(installed_parts))) + required_parts.extend([0] * (max_len - len(required_parts))) + print(f"[DEBUG] {package_name}: padded parts - installed={installed_parts}, required={required_parts}") + + # Check major.minor compatibility, allow newer patch versions + if len(installed_parts) >= 2 and len(required_parts) >= 2: + # Major and minor versions must match + major_minor_match = (installed_parts[0] == required_parts[0] and + installed_parts[1] == required_parts[1]) + + if major_minor_match: + # Patch version can be equal or higher + if len(installed_parts) >= 3 and len(required_parts) >= 3: + patch_ok = installed_parts[2] >= required_parts[2] + return patch_ok + return True + return False + except ValueError as e: + # If version parsing fails, fall back to exact match + exact_match = installed_base_version == required_base_version + return exact_match + else: + # For other packages, require exact version match + exact_match = installed_base_version == required_base_version + return exact_match + + except (importlib.metadata.PackageNotFoundError, Exception) as e: + return False + def install_remote_packages(package_names): for package_name in package_names: + original_package_name = package_name + required_version = None + if package_name in build_config.remote_packages: package_attr = build_config.remote_packages[package_name] if hasattr(package_attr, "version"): - package_name += f"=={package_attr.version}" + required_version = package_attr.version + package_name += f"=={required_version}" + + # Check if package is already installed with the correct version + if is_package_installed(original_package_name, required_version): + print(f"Package {original_package_name} (version {required_version or 'any'}) is already installed, skipping...") + continue + + print(f"Installing {original_package_name} version {required_version or 'latest'}...") + if hasattr(package_attr, "url"): url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url" @@ -152,5 +226,34 @@ def install_remote_packages(package_names): package_name, url_option, package_attr.url ]) continue + else: + # Check if package is already installed + if is_package_installed(original_package_name): + print(f"Package {original_package_name} is already installed, skipping...") + continue + + print(f"Installing {original_package_name}...") - subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package_name]) \ No newline at end of file + subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package_name]) + +def install_platform_packages(): + if hasattr(build_config, 'platform_packages') and OS_TYPE in build_config.platform_packages: + packages = build_config.platform_packages[OS_TYPE] + for package in packages: + # Extract package name (without version constraints) + package_name = package.split('==')[0].split('>=')[0].split('<=')[0].split('<')[0].split('>')[0] + + if is_package_installed(package_name): + print(f"Platform package {package_name} is already installed, skipping...") + continue + + print(f"Installing platform package {package}...") + subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package]) + +def wheels_dir_exists_and_not_empty(builds_dir): + if not os.path.exists(builds_dir): + return False + + # Check if directory has any .whl files + wheel_files = glob.glob(os.path.join(builds_dir, "**/*.whl"), recursive=True) + return len(wheel_files) > 0 \ No newline at end of file diff --git a/_Pre_Builds/_Build_Scripts/dependencies.txt b/_Pre_Builds/_Build_Scripts/dependencies.txt index a7c08db..7244097 100644 --- a/_Pre_Builds/_Build_Scripts/dependencies.txt +++ b/_Pre_Builds/_Build_Scripts/dependencies.txt @@ -1,3 +1,4 @@ +https://github.com/traveller59/spconv.git https://github.com/ashawkey/diff-gaussian-rasterization.git https://github.com/NVlabs/nvdiffrast.git https://github.com/ashawkey/kiuikit.git diff --git a/install.py b/install.py index 0fb4124..e2c0c40 100644 --- a/install.py +++ b/install.py @@ -20,6 +20,8 @@ try: get_platform_config_name, git_folder_parallel, install_remote_packages, + install_platform_packages, + wheels_dir_exists_and_not_empty, build_config, PYTHON_PATH, WHEELS_ROOT_ABS_PATH, @@ -27,6 +29,12 @@ try: ) from shared_utils.log_utils import cstr + # Ensure PyGithub is installed for downloading wheels + try: + import github + except ImportError: + subprocess.run([sys.executable, "-m", "pip", "install", "PyGithub"]) + def try_auto_build_all(builds_dir): cstr(f"Try building all required packages...").msg.print() result = subprocess.run( @@ -48,6 +56,7 @@ try: # Install packages that needs specify remote url install_remote_packages(build_config.build_base_packages) + install_platform_packages() # Get the target remote pre-built wheels directory name and path platform_config_name = get_platform_config_name() @@ -56,8 +65,12 @@ try: builds_dir = os.path.join(WHEELS_ROOT_ABS_PATH, platform_config_name) build_succeed = False + # Check if wheels already exist locally + if wheels_dir_exists_and_not_empty(builds_dir): + cstr(f"Found existing wheels in {builds_dir}").msg.print() + build_succeed = True # Download pre-build wheels if exist - if git_folder_parallel(build_config.repo_id, remote_builds_dir_name, recursive=True, root_outdir=builds_dir): + elif git_folder_parallel(build_config.repo_id, remote_builds_dir_name, recursive=True, root_outdir=builds_dir): build_succeed = True # Build the wheels if couldn't find pre-build wheels elif try_auto_build_all(builds_dir): diff --git a/requirements.txt b/requirements.txt index 1d6098b..8b6c13d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # build tools cmake ninja +pccm # web tools PyGithub @@ -15,13 +16,17 @@ opencv-python pillow nerfacc>=0.5.3 slangtorch +cumm==0.7.11 # for spconv + git+https://github.com/EasternJournalist/utils3d.git#egg=utils3d +git+https://github.com/ashawkey/kiuikit.git#egg=kiuikit # for use ML models diffusers>=0.26.1 transformers>=4.36.2 safetensors open_clip_torch +mmgp # for training differentiable tensors torchmetrics