diff --git a/_Pre_Builds/_Build_Scripts/auto_build_all.py b/_Pre_Builds/_Build_Scripts/auto_build_all.py index 09e084f..69500ce 100644 --- a/_Pre_Builds/_Build_Scripts/auto_build_all.py +++ b/_Pre_Builds/_Build_Scripts/auto_build_all.py @@ -4,6 +4,7 @@ from os.path import dirname import subprocess import re import time +import platform BUILD_SCRIPT_ROOT_ABS_PATH = dirname(__file__) sys.path.append(BUILD_SCRIPT_ROOT_ABS_PATH) @@ -51,6 +52,11 @@ def setup_build_env(): install_remote_packages(build_config.build_base_packages) install_platform_packages() + + # Install spconv with correct CUDA version + from build_utils import install_spconv + if not install_spconv(): + raise RuntimeError("Failed to install spconv") def clone_or_update_read_dependency(dependency_url, dependency_dir): if os.path.exists(dependency_dir): @@ -61,19 +67,71 @@ def clone_or_update_read_dependency(dependency_url, dependency_dir): subprocess.run(["git", "clone", "--recursive", dependency_url, dependency_dir]) def build_python_wheel(dependency_dir, output_dir): - # Build wheel and move the wheel file we just built to the output directory - print(f"Building {dependency_dir}") + """Build wheel using available build method""" + print(f"Checking {dependency_dir}") + + setup_py_path = os.path.join(dependency_dir, "setup.py") + pyproject_toml_path = os.path.join(dependency_dir, "pyproject.toml") + + has_setup_py = os.path.exists(setup_py_path) + has_pyproject = os.path.exists(pyproject_toml_path) + + if not has_setup_py and not has_pyproject: + print(f"No setup.py or pyproject.toml found in {dependency_dir}, skipping") + return None # Skipped + + # Try different build methods + if has_setup_py: + print(f"Building wheel for {dependency_dir} using setup.py") + result = subprocess.run( + [PYTHON_PATH, "setup.py", "bdist_wheel", "--dist-dir", output_dir], + cwd=dependency_dir, text=True, capture_output=True + ) + else: + print(f"Building wheel for {dependency_dir} using pip wheel") + result = subprocess.run( + [PYTHON_PATH, "-m", "pip", "wheel", ".", "--no-deps", "--wheel-dir", output_dir], + cwd=dependency_dir, text=True, capture_output=True + ) - result = subprocess.run([PYTHON_PATH, "setup.py", "bdist_wheel", "--dist-dir", output_dir], cwd=dependency_dir, text=True, capture_output=True) - #print(f"returncode: {result.returncode} \n\n Output: {result.stdout} \n\n Error: {result.stderr}") build_failed = result.returncode != 0 if build_failed: print(f"[Wheel BUILD LOG]\n{result.stdout}") print(f"[Wheel BUILD ERROR LOG]\n{result.stderr}") + + # If first method failed and we have both files, try the other method + if has_setup_py and has_pyproject: + print(f"Retrying with pip wheel method...") + result = subprocess.run( + [PYTHON_PATH, "-m", "pip", "wheel", ".", "--no-deps", "--wheel-dir", output_dir], + cwd=dependency_dir, text=True, capture_output=True + ) + build_failed = result.returncode != 0 + + if build_failed: + print(f"[Retry BUILD LOG]\n{result.stdout}") + print(f"[Retry BUILD ERROR LOG]\n{result.stderr}") - print(f" Build {dependency_dir} {'Failed' if build_failed else 'Succeed'}") - return build_failed + # If wheel building failed, try direct installation as fallback + if build_failed: + print(f"Wheel building failed, trying direct installation...") + result = subprocess.run( + [PYTHON_PATH, "-m", "pip", "install", "."], + cwd=dependency_dir, text=True, capture_output=True + ) + + if result.returncode == 0: + print(f"Direct install {dependency_dir} Succeed (no wheel created)") + return "installed" # Installed directly, no wheel + else: + print(f"[Direct INSTALL LOG]\n{result.stdout}") + print(f"[Direct INSTALL ERROR LOG]\n{result.stderr}") + print(f"Build and install {dependency_dir} Failed") + return False # Failed + else: + print(f"Build {dependency_dir} Succeed") + return True # Success def main(args): start_time = time.time() @@ -95,17 +153,39 @@ def main(args): # Build all dependencies and move them to output_root_path failed_build = [] + skipped_build = [] + successful_build = [] + direct_installed = [] for dependency in dependencies: dependency_dir, is_url = get_dependency_dir(dependency) if is_url: clone_or_update_read_dependency(dependency, dependency_dir) + + # Check if directory exists + if not os.path.exists(dependency_dir): + print(f"Directory {dependency_dir} does not exist, skipping") + skipped_build.append(dependency) + continue - build_failed = build_python_wheel(dependency_dir, output_root_path) - if build_failed: + result = build_python_wheel(dependency_dir, output_root_path) + if result is None: + skipped_build.append(dependency) + elif result == "installed": + direct_installed.append(dependency) + elif result is False: failed_build.append(dependency) + else: + successful_build.append(dependency) hours, minutes, seconds = calculate_runtime(start_time) print(f"Build all dependencies finished in {int(hours):0>2}:{int(minutes):0>2}:{seconds:05.2f}") + print(f"Results: {len(successful_build)} wheels built, {len(direct_installed)} direct installs, {len(skipped_build)} skipped, {len(failed_build)} failed") + + if len(skipped_build) > 0: + print(f"Skipped dependencies (no setup.py/pyproject.toml): {skipped_build}") + + if len(direct_installed) > 0: + print(f"Direct installed dependencies (no wheel): {direct_installed}") if len(failed_build) == 0: print(f"[Comfy3D BUILD SUCCEED]") diff --git a/_Pre_Builds/_Build_Scripts/build_config.yaml b/_Pre_Builds/_Build_Scripts/build_config.yaml index daea55d..a7294fb 100644 --- a/_Pre_Builds/_Build_Scripts/build_config.yaml +++ b/_Pre_Builds/_Build_Scripts/build_config.yaml @@ -1,4 +1,4 @@ -repo_id: "MrForExample/Comfy3D_Pre_Builds" +repo_id: "DenisKochetov/Comfy3D_Pre_Builds" supported_cuda_versions: ["12.8", "12.6", "12.4", "12.1", "11.8"] cuda_version: "12.8" @@ -22,6 +22,14 @@ platform_packages: linux: - "triton" +# Simple spconv installation with CUDA mapping +spconv_cuda_mapping: + "12.8": "cu126" # Default for 12.8 and maximum available + "12.6": "cu126" + "12.4": "cu124" + "12.1": "cu121" + "11.8": "cu118" + # All packages will be automatically installed at beginning of install.py follwing orders from top to bottom remote_packages: torch: @@ -36,7 +44,6 @@ remote_packages: version: "0.0.30" url: "https://download.pytorch.org/whl/${cuda_version}" - # At the time of writing, torch-scatter is not available for PyTorch 2.5.0 - #torch-scatter: - # url_option: "--find-links" - # url: "https://data.pyg.org/whl/torch-${remote_packages.torch.version}+${cuda_version}.html" \ No newline at end of file + torch-scatter: + url_option: "--find-links" + url: "https://data.pyg.org/whl/torch-${remote_packages.torch.version}+${cuda_version}.html" \ No newline at end of file diff --git a/_Pre_Builds/_Build_Scripts/build_utils.py b/_Pre_Builds/_Build_Scripts/build_utils.py index 559cb22..aa44165 100644 --- a/_Pre_Builds/_Build_Scripts/build_utils.py +++ b/_Pre_Builds/_Build_Scripts/build_utils.py @@ -250,6 +250,50 @@ def install_platform_packages(): print(f"Installing platform package {package}...") subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package]) +def install_spconv(): + """Simple spconv installation with correct CUDA version""" + # Check if spconv is already installed + try: + import spconv.core_cc + print("spconv is already installed and working") + return True + except ImportError: + pass + + # Get CUDA version mapping + if not hasattr(build_config, 'spconv_cuda_mapping'): + print("No spconv_cuda_mapping in config, skipping spconv installation") + return True + + # Convert CUDA_VERSION (e.g. cu128 -> 12.8) for lookup + cuda_version_for_lookup = CUDA_VERSION[2:3] + '.' + CUDA_VERSION[3:] + cuda_suffix = build_config.spconv_cuda_mapping.get(cuda_version_for_lookup) + + if not cuda_suffix: + print(f"No spconv mapping for CUDA {cuda_version_for_lookup}, skipping") + return True + + spconv_package = f"spconv-{cuda_suffix}" + print(f"Installing {spconv_package}...") + + result = subprocess.run( + [PYTHON_PATH, "-m", "pip", "install", spconv_package], + text=True, capture_output=True + ) + + if result.returncode != 0: + print(f"Failed to install {spconv_package}: {result.stderr}") + return False + + # Verify installation + try: + import spconv.core_cc + print(f"Successfully installed and verified {spconv_package}") + return True + except ImportError as e: + print(f"spconv installed but verification failed: {e}") + return False + def wheels_dir_exists_and_not_empty(builds_dir): if not os.path.exists(builds_dir): return False diff --git a/install.py b/install.py index e2c0c40..e81467f 100644 --- a/install.py +++ b/install.py @@ -7,6 +7,7 @@ from os.path import dirname import glob import subprocess import traceback +import platform if sys.argv[0] == 'install.py': sys.path.append('.') # for portable version @@ -21,6 +22,7 @@ try: git_folder_parallel, install_remote_packages, install_platform_packages, + install_spconv, wheels_dir_exists_and_not_empty, build_config, PYTHON_PATH, @@ -34,9 +36,65 @@ try: import github except ImportError: subprocess.run([sys.executable, "-m", "pip", "install", "PyGithub"]) - + + def install_local_wheels(builds_dir): + """Install all wheels from local directory""" + wheel_files = glob.glob(os.path.join(builds_dir, "**/*.whl"), recursive=True) + if not wheel_files: + cstr("No wheel files found in directory").warning.print() + return False + + success_count = 0 + for wheel_path in wheel_files: + result = subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", "--no-deps", "--force-reinstall", wheel_path], + text=True, capture_output=True) + if result.returncode == 0: + cstr(f"Successfully installed wheel: {os.path.basename(wheel_path)}").msg.print() + success_count += 1 + else: + cstr(f"Failed to install wheel {os.path.basename(wheel_path)}: {result.stderr}").error.print() + + if success_count == len(wheel_files): + cstr(f"Successfully installed all {len(wheel_files)} wheels").msg.print() + return True + elif success_count > 0: + cstr(f"Partially successful: {success_count}/{len(wheel_files)} wheels installed").warning.print() + return False + else: + cstr("Failed to install any wheels").error.print() + return False + + def try_wheels_first_approach(): + """Try wheels-first approach for all platforms""" + platform_config_name = get_platform_config_name() + builds_dir = os.path.join(WHEELS_ROOT_ABS_PATH, platform_config_name) + + cstr("Trying wheels-first approach...").msg.print() + wheels_installed = False + + # Check existing wheels + if wheels_dir_exists_and_not_empty(builds_dir): + cstr(f"Found existing wheels in {builds_dir}").msg.print() + if install_local_wheels(builds_dir): + wheels_installed = True + cstr("Installed wheels from local directory").msg.print() + + # Try downloading wheels from repository if not found locally + if not wheels_installed: + remote_builds_dir_name = f"{build_config.wheels_dir_name}/{platform_config_name}" + if git_folder_parallel(build_config.repo_id, remote_builds_dir_name, recursive=True, root_outdir=builds_dir): + cstr("Downloaded wheels from repository").msg.print() + if install_local_wheels(builds_dir): + wheels_installed = True + cstr("Installed wheels from repository").msg.print() + else: + cstr("Could not download wheels from repository").warning.print() + + return wheels_installed + def try_auto_build_all(builds_dir): cstr(f"Try building all required packages...").msg.print() + result = subprocess.run( [PYTHON_PATH, "auto_build_all.py", "--output_root_dir", builds_dir], cwd=BUILD_SCRIPT_ROOT_ABS_PATH, text=True, capture_output=True @@ -49,38 +107,66 @@ try: return build_succeed - def install_local_wheels(builds_dir): - for wheel_path in glob.glob(os.path.join(builds_dir, "**/*.whl"), recursive=True): - subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", "--no-deps", "--force-reinstall", wheel_path]) - cstr(f"pip install {wheel_path} to {PYTHON_PATH}").msg.print() - # Install packages that needs specify remote url install_remote_packages(build_config.build_base_packages) install_platform_packages() + # Check and install build tools if needed + cstr("Checking build tools...").msg.print() + build_tools = ["ninja", "cmake", "setuptools", "wheel"] + for tool in build_tools: + try: + __import__(tool) + cstr(f"{tool} is already installed").msg.print() + except ImportError: + cstr(f"Installing {tool}...").msg.print() + result = subprocess.run( + [PYTHON_PATH, "-m", "pip", "install", "--upgrade", tool], + text=True, capture_output=True + ) + if result.returncode != 0: + cstr(f"[{tool} INSTALL ERROR]\n{result.stderr}").error.print() + raise RuntimeError(f"Failed to install {tool}") + + # Main installation logic + spconv_success = False + wheels_success = False + # Get the target remote pre-built wheels directory name and path platform_config_name = get_platform_config_name() - remote_builds_dir_name = f"{build_config.wheels_dir_name}/{platform_config_name}" - # Get the directory path which wheels will be downloaded or build into it builds_dir = os.path.join(WHEELS_ROOT_ABS_PATH, platform_config_name) - build_succeed = False - # Check if wheels already exist locally - if wheels_dir_exists_and_not_empty(builds_dir): - cstr(f"Found existing wheels in {builds_dir}").msg.print() - build_succeed = True - # Download pre-build wheels if exist - elif git_folder_parallel(build_config.repo_id, remote_builds_dir_name, recursive=True, root_outdir=builds_dir): - build_succeed = True - # Build the wheels if couldn't find pre-build wheels - elif try_auto_build_all(builds_dir): - build_succeed = True - - if build_succeed: - install_local_wheels(builds_dir) - cstr("Successfully installed all required wheels").msg.print() + # Unified wheels-first approach for all platforms + cstr("Starting unified installation process...").msg.print() + + # Step 1: Try wheels first + wheels_success = try_wheels_first_approach() + + # Step 2: Install missing packages (like spconv) + spconv_success = install_spconv() + + # Step 3: If wheels failed, try building + if not wheels_success: + cstr("Wheels installation failed, trying to build from source...").warning.print() + if try_auto_build_all(builds_dir): + install_local_wheels(builds_dir) + wheels_success = True + cstr("Successfully built and installed wheels").msg.print() + else: + cstr("Building wheels also failed").error.print() + + # Final verification + if not install_spconv(): + raise RuntimeError("spconv failed final verification. Please check the installation.") + + if spconv_success: + cstr("Successfully installed spconv").msg.print() + if wheels_success: + cstr("Successfully installed all available wheels").msg.print() + else: + cstr("Note: Some wheels could not be installed, but this is not critical").warning.print() else: - raise RuntimeError("Comfy3D build failed") + raise RuntimeError("Failed to install spconv") # Download python cpp source files for current python environment remote_pycpp_dir_name = f"_Python_Source_cpp/{PYTHON_VERSION}" diff --git a/requirements.txt b/requirements.txt index 8b6c13d..b69a82d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,6 +55,7 @@ pyvista pymeshfix igraph diso +open3d # configs & extra torchtyping>=0.1.5