nodes/MrForExample-ComfyUI-3D-Pack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #494 from DenisKochetov/linux-install

Browse Source 
updated wheels installation
This commit is contained in:
Mr. For Example
2025-06-15 15:13:42 +01:00
committed by GitHub
parent faa925cd16 b284bf9eb2
commit fe2c9ce5bc
5 changed files with 284 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_Pre_Builds/_Build_Scripts/auto_build_all.py
+88 -8
View File
@@ -4,6 +4,7 @@ from os.path import dirname
import subprocess
import re
import time
import platform
BUILD_SCRIPT_ROOT_ABS_PATH = dirname(__file__)
sys.path.append(BUILD_SCRIPT_ROOT_ABS_PATH)
@@ -51,6 +52,11 @@ def setup_build_env():
install_remote_packages(build_config.build_base_packages)
install_platform_packages()
# Install spconv with correct CUDA version
from build_utils import install_spconv
if not install_spconv():
raise RuntimeError("Failed to install spconv")
def clone_or_update_read_dependency(dependency_url, dependency_dir):
if os.path.exists(dependency_dir):
@@ -61,19 +67,71 @@ def clone_or_update_read_dependency(dependency_url, dependency_dir):
subprocess.run(["git", "clone", "--recursive", dependency_url, dependency_dir])
def build_python_wheel(dependency_dir, output_dir):
# Build wheel and move the wheel file we just built to the output directory
print(f"Building {dependency_dir}")
"""Build wheel using available build method"""
print(f"Checking {dependency_dir}")
setup_py_path = os.path.join(dependency_dir, "setup.py")
pyproject_toml_path = os.path.join(dependency_dir, "pyproject.toml")
has_setup_py = os.path.exists(setup_py_path)
has_pyproject = os.path.exists(pyproject_toml_path)
if not has_setup_py and not has_pyproject:
print(f"No setup.py or pyproject.toml found in {dependency_dir}, skipping")
return None # Skipped
# Try different build methods
if has_setup_py:
print(f"Building wheel for {dependency_dir} using setup.py")
result = subprocess.run(
[PYTHON_PATH, "setup.py", "bdist_wheel", "--dist-dir", output_dir],
cwd=dependency_dir, text=True, capture_output=True
)
else:
print(f"Building wheel for {dependency_dir} using pip wheel")
result = subprocess.run(
[PYTHON_PATH, "-m", "pip", "wheel", ".", "--no-deps", "--wheel-dir", output_dir],
cwd=dependency_dir, text=True, capture_output=True
)
result = subprocess.run([PYTHON_PATH, "setup.py", "bdist_wheel", "--dist-dir", output_dir], cwd=dependency_dir, text=True, capture_output=True)
#print(f"returncode: {result.returncode} \n\n Output: {result.stdout} \n\n Error: {result.stderr}")
build_failed = result.returncode != 0
if build_failed:
print(f"[Wheel BUILD LOG]\n{result.stdout}")
print(f"[Wheel BUILD ERROR LOG]\n{result.stderr}")
# If first method failed and we have both files, try the other method
if has_setup_py and has_pyproject:
print(f"Retrying with pip wheel method...")
result = subprocess.run(
[PYTHON_PATH, "-m", "pip", "wheel", ".", "--no-deps", "--wheel-dir", output_dir],
cwd=dependency_dir, text=True, capture_output=True
)
build_failed = result.returncode != 0
if build_failed:
print(f"[Retry BUILD LOG]\n{result.stdout}")
print(f"[Retry BUILD ERROR LOG]\n{result.stderr}")
print(f" Build {dependency_dir} {'Failed' if build_failed else 'Succeed'}")
return build_failed
# If wheel building failed, try direct installation as fallback
if build_failed:
print(f"Wheel building failed, trying direct installation...")
result = subprocess.run(
[PYTHON_PATH, "-m", "pip", "install", "."],
cwd=dependency_dir, text=True, capture_output=True
)
if result.returncode == 0:
print(f"Direct install {dependency_dir} Succeed (no wheel created)")
return "installed" # Installed directly, no wheel
else:
print(f"[Direct INSTALL LOG]\n{result.stdout}")
print(f"[Direct INSTALL ERROR LOG]\n{result.stderr}")
print(f"Build and install {dependency_dir} Failed")
return False # Failed
else:
print(f"Build {dependency_dir} Succeed")
return True # Success
def main(args):
start_time = time.time()
@@ -95,17 +153,39 @@ def main(args):
# Build all dependencies and move them to output_root_path
failed_build = []
skipped_build = []
successful_build = []
direct_installed = []
for dependency in dependencies:
dependency_dir, is_url = get_dependency_dir(dependency)
if is_url:
clone_or_update_read_dependency(dependency, dependency_dir)
# Check if directory exists
if not os.path.exists(dependency_dir):
print(f"Directory {dependency_dir} does not exist, skipping")
skipped_build.append(dependency)
continue
build_failed = build_python_wheel(dependency_dir, output_root_path)
if build_failed:
result = build_python_wheel(dependency_dir, output_root_path)
if result is None:
skipped_build.append(dependency)
elif result == "installed":
direct_installed.append(dependency)
elif result is False:
failed_build.append(dependency)
else:
successful_build.append(dependency)
hours, minutes, seconds = calculate_runtime(start_time)
print(f"Build all dependencies finished in {int(hours):0>2}:{int(minutes):0>2}:{seconds:05.2f}")
print(f"Results: {len(successful_build)} wheels built, {len(direct_installed)} direct installs, {len(skipped_build)} skipped, {len(failed_build)} failed")
if len(skipped_build) > 0:
print(f"Skipped dependencies (no setup.py/pyproject.toml): {skipped_build}")
if len(direct_installed) > 0:
print(f"Direct installed dependencies (no wheel): {direct_installed}")
if len(failed_build) == 0:
print(f"[Comfy3D BUILD SUCCEED]")
_Pre_Builds/_Build_Scripts/build_config.yaml
+12 -5
View File
@@ -1,4 +1,4 @@
repo_id: "MrForExample/Comfy3D_Pre_Builds"
repo_id: "DenisKochetov/Comfy3D_Pre_Builds"
supported_cuda_versions: ["12.8", "12.6", "12.4", "12.1", "11.8"]
cuda_version: "12.8"
@@ -22,6 +22,14 @@ platform_packages:
linux:
- "triton"
# Simple spconv installation with CUDA mapping
spconv_cuda_mapping:
"12.8": "cu126" # Default for 12.8 and maximum available
"12.6": "cu126"
"12.4": "cu124"
"12.1": "cu121"
"11.8": "cu118"
# All packages will be automatically installed at beginning of install.py follwing orders from top to bottom
remote_packages:
torch:
@@ -36,7 +44,6 @@ remote_packages:
version: "0.0.30"
url: "https://download.pytorch.org/whl/${cuda_version}"
# At the time of writing, torch-scatter is not available for PyTorch 2.5.0
#torch-scatter:
# url_option: "--find-links"
# url: "https://data.pyg.org/whl/torch-${remote_packages.torch.version}+${cuda_version}.html"
torch-scatter:
url_option: "--find-links"
url: "https://data.pyg.org/whl/torch-${remote_packages.torch.version}+${cuda_version}.html"
_Pre_Builds/_Build_Scripts/build_utils.py
+63
View File
@@ -250,6 +250,69 @@ def install_platform_packages():
print(f"Installing platform package {package}...")
subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package])
def install_spconv():
"""Simple spconv installation with correct CUDA version"""
# Check if spconv is already installed and working
try:
import spconv.core_cc
print("spconv is already installed and working")
return True
except ImportError:
pass
# Get CUDA version mapping
if not hasattr(build_config, 'spconv_cuda_mapping'):
print("No spconv_cuda_mapping in config, skipping spconv installation")
return True
# Fix CUDA version parsing: cu128 -> 12.8, cu121 -> 12.1, cu118 -> 11.8
if len(CUDA_VERSION) >= 5: # e.g., cu128, cu121
major = CUDA_VERSION[2:-1] # cu128 -> 12, cu121 -> 12, cu118 -> 11
minor = CUDA_VERSION[-1] # cu128 -> 8, cu121 -> 1, cu118 -> 8
cuda_version_for_lookup = f"{major}.{minor}"
else:
print(f"Invalid CUDA_VERSION format: {CUDA_VERSION}")
return False
print(f"Detected CUDA version: {cuda_version_for_lookup}")
cuda_suffix = build_config.spconv_cuda_mapping.get(cuda_version_for_lookup)
if not cuda_suffix:
print(f"No spconv mapping for CUDA {cuda_version_for_lookup}, skipping")
return True
spconv_package = f"spconv-{cuda_suffix}"
print(f"Installing {spconv_package}...")
result = subprocess.run(
[PYTHON_PATH, "-m", "pip", "install", spconv_package],
text=True, capture_output=True
)
if result.returncode != 0:
print(f"Failed to install {spconv_package}: {result.stderr}")
return False
# Verify installation
try:
import spconv.core_cc
print(f"Successfully installed and verified {spconv_package}")
return True
except ImportError as e:
print(f"spconv installed but verification failed: {e}")
return False
def check_spconv_after_wheels():
"""Check if spconv works after wheels installation"""
try:
import spconv.core_cc
print("spconv is working after wheels installation")
return True
except ImportError:
print("spconv not working, will install from mapping")
return False
def wheels_dir_exists_and_not_empty(builds_dir):
if not os.path.exists(builds_dir):
return False
install.py
+120 -24
View File
@@ -7,6 +7,7 @@ from os.path import dirname
import glob
import subprocess
import traceback
import platform
if sys.argv[0] == 'install.py':
sys.path.append('.') # for portable version
@@ -21,6 +22,8 @@ try:
git_folder_parallel,
install_remote_packages,
install_platform_packages,
install_spconv,
check_spconv_after_wheels,
wheels_dir_exists_and_not_empty,
build_config,
PYTHON_PATH,
@@ -34,9 +37,65 @@ try:
import github
except ImportError:
subprocess.run([sys.executable, "-m", "pip", "install", "PyGithub"])
def install_local_wheels(builds_dir):
"""Install all wheels from local directory"""
wheel_files = glob.glob(os.path.join(builds_dir, "**/*.whl"), recursive=True)
if not wheel_files:
cstr("No wheel files found in directory").warning.print()
return False
success_count = 0
for wheel_path in wheel_files:
result = subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", "--no-deps", "--force-reinstall", wheel_path],
text=True, capture_output=True)
if result.returncode == 0:
cstr(f"Successfully installed wheel: {os.path.basename(wheel_path)}").msg.print()
success_count += 1
else:
cstr(f"Failed to install wheel {os.path.basename(wheel_path)}: {result.stderr}").error.print()
if success_count == len(wheel_files):
cstr(f"Successfully installed all {len(wheel_files)} wheels").msg.print()
return True
elif success_count > 0:
cstr(f"Partially successful: {success_count}/{len(wheel_files)} wheels installed").warning.print()
return False
else:
cstr("Failed to install any wheels").error.print()
return False
def try_wheels_first_approach():
"""Try wheels-first approach for all platforms"""
platform_config_name = get_platform_config_name()
builds_dir = os.path.join(WHEELS_ROOT_ABS_PATH, platform_config_name)
cstr("Trying wheels-first approach...").msg.print()
wheels_installed = False
# Check existing wheels
if wheels_dir_exists_and_not_empty(builds_dir):
cstr(f"Found existing wheels in {builds_dir}").msg.print()
if install_local_wheels(builds_dir):
wheels_installed = True
cstr("Installed wheels from local directory").msg.print()
# Try downloading wheels from repository if not found locally
if not wheels_installed:
remote_builds_dir_name = f"{build_config.wheels_dir_name}/{platform_config_name}"
if git_folder_parallel(build_config.repo_id, remote_builds_dir_name, recursive=True, root_outdir=builds_dir):
cstr("Downloaded wheels from repository").msg.print()
if install_local_wheels(builds_dir):
wheels_installed = True
cstr("Installed wheels from repository").msg.print()
else:
cstr("Could not download wheels from repository").warning.print()
return wheels_installed
def try_auto_build_all(builds_dir):
cstr(f"Try building all required packages...").msg.print()
result = subprocess.run(
[PYTHON_PATH, "auto_build_all.py", "--output_root_dir", builds_dir],
cwd=BUILD_SCRIPT_ROOT_ABS_PATH, text=True, capture_output=True
@@ -49,38 +108,75 @@ try:
return build_succeed
def install_local_wheels(builds_dir):
for wheel_path in glob.glob(os.path.join(builds_dir, "**/*.whl"), recursive=True):
subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", "--no-deps", "--force-reinstall", wheel_path])
cstr(f"pip install {wheel_path} to {PYTHON_PATH}").msg.print()
# Install packages that needs specify remote url
install_remote_packages(build_config.build_base_packages)
install_platform_packages()
# Check and install build tools if needed
cstr("Checking build tools...").msg.print()
build_tools = ["ninja", "cmake", "setuptools", "wheel"]
for tool in build_tools:
try:
__import__(tool)
cstr(f"{tool} is already installed").msg.print()
except ImportError:
cstr(f"Installing {tool}...").msg.print()
result = subprocess.run(
[PYTHON_PATH, "-m", "pip", "install", "--upgrade", tool],
text=True, capture_output=True
)
if result.returncode != 0:
cstr(f"[{tool} INSTALL ERROR]\n{result.stderr}").error.print()
raise RuntimeError(f"Failed to install {tool}")
# Main installation logic
spconv_success = False
wheels_success = False
# Get the target remote pre-built wheels directory name and path
platform_config_name = get_platform_config_name()
remote_builds_dir_name = f"{build_config.wheels_dir_name}/{platform_config_name}"
# Get the directory path which wheels will be downloaded or build into it
builds_dir = os.path.join(WHEELS_ROOT_ABS_PATH, platform_config_name)
build_succeed = False
# Check if wheels already exist locally
if wheels_dir_exists_and_not_empty(builds_dir):
cstr(f"Found existing wheels in {builds_dir}").msg.print()
build_succeed = True
# Download pre-build wheels if exist
elif git_folder_parallel(build_config.repo_id, remote_builds_dir_name, recursive=True, root_outdir=builds_dir):
build_succeed = True
# Build the wheels if couldn't find pre-build wheels
elif try_auto_build_all(builds_dir):
build_succeed = True
if build_succeed:
install_local_wheels(builds_dir)
cstr("Successfully installed all required wheels").msg.print()
# Unified wheels-first approach for all platforms
cstr("Starting unified installation process...").msg.print()
# Step 1: Try wheels first
wheels_success = try_wheels_first_approach()
# Step 2: Check if spconv works after wheels, install if needed
if wheels_success and check_spconv_after_wheels():
spconv_success = True
else:
raise RuntimeError("Comfy3D build failed")
# Install spconv using CUDA mapping
spconv_success = install_spconv()
# Step 3: If wheels failed, try building
if not wheels_success:
cstr("Wheels installation failed, trying to build from source...").warning.print()
if try_auto_build_all(builds_dir):
install_local_wheels(builds_dir)
wheels_success = True
cstr("Successfully built and installed wheels").msg.print()
# Check spconv again after building
if not check_spconv_after_wheels():
spconv_success = install_spconv()
else:
spconv_success = True
else:
cstr("Building wheels also failed").error.print()
# Final verification
if not install_spconv():
raise RuntimeError("spconv failed final verification. Please check the installation.")
if spconv_success:
cstr("Successfully installed spconv").msg.print()
if wheels_success:
cstr("Successfully installed all available wheels").msg.print()
else:
cstr("Note: Some wheels could not be installed, but this is not critical").warning.print()
else:
raise RuntimeError("Failed to install spconv")
# Download python cpp source files for current python environment
remote_pycpp_dir_name = f"_Python_Source_cpp/{PYTHON_VERSION}"
requirements.txt
+1
View File
@@ -55,6 +55,7 @@ pyvista
pymeshfix
igraph
diso
open3d
# configs & extra
torchtyping>=0.1.5