Create auto Install script & update docs
To support one click install for ComfyUI-Manager and cli, almost there!
This commit is contained in:
Mr. For Example
@@ -1,46 +1,34 @@
|
||||
import sys
|
||||
import os
|
||||
from os.path import dirname
|
||||
import glob
|
||||
import platform
|
||||
import subprocess
|
||||
import re
|
||||
import time
|
||||
|
||||
SCRIPT_ROOT_ABS_PATH = dirname(os.path.abspath(__file__))
|
||||
DEPENDENCIES_FILE_ABS_PATH = os.path.join(SCRIPT_ROOT_ABS_PATH, "dependencies.txt")
|
||||
BUILD_REQUIREMENTS_FILE_ABS_PATH = os.path.join(SCRIPT_ROOT_ABS_PATH, "build_requirements.txt")
|
||||
BUILD_ROOT_ABS_PATH = dirname(SCRIPT_ROOT_ABS_PATH)
|
||||
DEPENDENCIES_ROOT_ABS_PATH = os.path.join(BUILD_ROOT_ABS_PATH, "_Build_Dependencies")
|
||||
LIBS_ROOT_ABS_PATH = os.path.join(BUILD_ROOT_ABS_PATH, "_Libs")
|
||||
BUILD_SCRIPT_ROOT_ABS_PATH = dirname(__file__)
|
||||
sys.path.append(BUILD_SCRIPT_ROOT_ABS_PATH)
|
||||
|
||||
def get_parent_dirpath_n_level_up(abs_path, n=1):
|
||||
for i in range(n):
|
||||
abs_path = dirname(abs_path)
|
||||
return abs_path
|
||||
|
||||
COMFY_PYTHON_ABS_PATH = os.path.join(get_parent_dirpath_n_level_up(BUILD_ROOT_ABS_PATH, 4), "python_embeded", "python")
|
||||
|
||||
def get_platform_config_name(cuda_version):
|
||||
platform_config_name = "_Wheels"
|
||||
|
||||
# Add OS Type
|
||||
if platform.system() == 'Windows':
|
||||
platform_config_name += "_win"
|
||||
elif platform.system() == "Linux":
|
||||
platform_config_name += "_linux"
|
||||
else:
|
||||
raise NotImplementedError(f"Platform {platform.system()} not supported!")
|
||||
|
||||
# Add Python Version, only first two version numbers, e.g. 3.12.4 -> 312
|
||||
platform_config_name += "_py" + "".join(platform.python_version().split('.')[:-1])
|
||||
|
||||
# Add CUDA Version
|
||||
platform_config_name += "_" + cuda_version
|
||||
|
||||
return platform_config_name
|
||||
from build_utils import (
|
||||
get_platform_config_name,
|
||||
calculate_runtime,
|
||||
git_folder_parallel,
|
||||
install_remote_packages,
|
||||
build_config,
|
||||
PYTHON_PATH,
|
||||
DEPENDENCIES_FILE_ABS_PATH,
|
||||
BUILD_REQUIREMENTS_FILE_ABS_PATH,
|
||||
DEPENDENCIES_ROOT_ABS_PATH,
|
||||
WHEELS_ROOT_ABS_PATH,
|
||||
LIBS_ROOT_ABS_PATH
|
||||
)
|
||||
|
||||
def read_dependencies(file_path):
|
||||
# Download required source libraries from remote
|
||||
if not os.path.exists(LIBS_ROOT_ABS_PATH):
|
||||
if not git_folder_parallel(build_config.repo_id, build_config.libs_dir_name, recursive=True, root_outdir=LIBS_ROOT_ABS_PATH):
|
||||
raise RuntimeError(f"Comfy3D install failed, couldn't download directory {build_config.libs_dir_name} in remote repository {build_config.repo_id}")
|
||||
dependencies = [ f.path for f in os.scandir(LIBS_ROOT_ABS_PATH) if f.is_dir() ]
|
||||
#dependencies = [] #ignore libraries for debug
|
||||
with open(file_path, "r") as f:
|
||||
dependencies += [line.strip() for line in f]
|
||||
|
||||
@@ -55,15 +43,12 @@ def get_dependency_dir(dependency):
|
||||
dependency_dir = os.path.join(LIBS_ROOT_ABS_PATH, dependency)
|
||||
return dependency_dir, is_url
|
||||
|
||||
def setup_build_env(python_path, args):
|
||||
def setup_build_env():
|
||||
# Set CMake argumens fo build packages based on CUDA
|
||||
os.environ["CMAKE_ARGS"]="-DBUILD_opencv_world=ON -DWITH_CUDA=ON -DCUDA_FAST_MATH=ON -DWITH_CUBLAS=ON -DCUDA_ARCH_PTX=9.0 -DWITH_NVCUVID=ON"
|
||||
subprocess.run([python_path, "-s", "-m", "pip", "install", "-r", BUILD_REQUIREMENTS_FILE_ABS_PATH])
|
||||
subprocess.run([
|
||||
python_path, "-s", "-m", "pip", "install",
|
||||
f"torch=={args.torch_version}", f"torchvision=={args.torchvision_version}",
|
||||
"--index-url", f"https://download.pytorch.org/whl/{args.cuda_version}"
|
||||
])
|
||||
subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", "-r", BUILD_REQUIREMENTS_FILE_ABS_PATH])
|
||||
|
||||
install_remote_packages(build_config.build_base_packages)
|
||||
|
||||
def clone_or_update_read_dependency(dependency_url, dependency_dir):
|
||||
if os.path.exists(dependency_dir):
|
||||
@@ -73,37 +58,53 @@ def clone_or_update_read_dependency(dependency_url, dependency_dir):
|
||||
# Clone the dependency
|
||||
subprocess.run(["git", "clone", "--recursive", dependency_url, dependency_dir])
|
||||
|
||||
def build_python_wheel(python_path, dependency_dir, output_dir):
|
||||
def build_python_wheel(dependency_dir, output_dir):
|
||||
# Build wheel and move the wheel file we just built to the output directory
|
||||
subprocess.run([python_path, "setup.py", "bdist_wheel", "--dist-dir", output_dir], cwd=dependency_dir)
|
||||
print(f"Building {dependency_dir}")
|
||||
|
||||
result = subprocess.run([PYTHON_PATH, "setup.py", "bdist_wheel", "--dist-dir", output_dir], cwd=dependency_dir, shell=True, text=True, capture_output=True)
|
||||
#print(f"returncode: {result.returncode} \n\n Output: {result.stdout} \n\n Error: {result.stderr}")
|
||||
build_failed = result.returncode != 0
|
||||
|
||||
print(f" Build {dependency_dir} {'Failed' if build_failed else 'Succeed'}")
|
||||
return build_failed
|
||||
|
||||
def main(args):
|
||||
# Get python executable from path
|
||||
python_path = sys.executable
|
||||
start_time = time.time()
|
||||
|
||||
# Create the output folder if it doesn't exist
|
||||
output_root_path = args.output_root_dir
|
||||
if output_root_path is None:
|
||||
output_root_path = os.path.join(BUILD_ROOT_ABS_PATH, get_platform_config_name(args.cuda_version))
|
||||
output_root_path = os.path.join(WHEELS_ROOT_ABS_PATH, get_platform_config_name())
|
||||
elif not os.path.isabs(output_root_path):
|
||||
output_root_path = os.path.join(BUILD_ROOT_ABS_PATH, output_root_path)
|
||||
output_root_path = os.path.join(WHEELS_ROOT_ABS_PATH, output_root_path)
|
||||
|
||||
os.makedirs(output_root_path, exist_ok=True)
|
||||
|
||||
# Read dependencies names from the text file
|
||||
dependencies = read_dependencies(DEPENDENCIES_FILE_ABS_PATH)
|
||||
print(dependencies)
|
||||
|
||||
# Setup environment variables and fundamental packages for build
|
||||
setup_build_env(python_path, args)
|
||||
setup_build_env()
|
||||
|
||||
# Read dependencies names from the text file
|
||||
dependencies = read_dependencies(DEPENDENCIES_FILE_ABS_PATH)
|
||||
|
||||
# Build all dependencies and move them to output_root_path
|
||||
failed_build = []
|
||||
for dependency in dependencies:
|
||||
dependency_dir, is_url = get_dependency_dir(dependency)
|
||||
if is_url:
|
||||
clone_or_update_read_dependency(dependency, dependency_dir)
|
||||
|
||||
build_python_wheel(python_path, dependency_dir, output_root_path)
|
||||
build_failed = build_python_wheel(dependency_dir, output_root_path)
|
||||
if build_failed:
|
||||
failed_build.append(dependency)
|
||||
|
||||
hours, minutes, seconds = calculate_runtime(start_time)
|
||||
print(f"Build all dependencies finished in {int(hours):0>2}:{int(minutes):0>2}:{seconds:05.2f}")
|
||||
|
||||
if len(failed_build) == 0:
|
||||
print(f"[Comfy3D BUILD SUCCEED]")
|
||||
else:
|
||||
raise RuntimeError(f"[Comfy3D BUILD FAILED]: Following dependencies failed to build: {failed_build}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
@@ -114,27 +115,6 @@ if __name__ == "__main__":
|
||||
default=None,
|
||||
help="Path to the target build directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cuda_version",
|
||||
type=str,
|
||||
default="cu121",
|
||||
choices="[cu121, cu118]",
|
||||
help="CUDA Toolkit version",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--torch_version",
|
||||
type=str,
|
||||
default="2.3.0",
|
||||
help="Pytorch version",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--torchvision_version",
|
||||
type=str,
|
||||
default="0.18.0",
|
||||
help="Pytorch vision library version",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args)
|
||||
|
||||
#C:\Users\reall\Softwares\Miniconda\envs\Comfy3D_Builds_py310_cu118\python _Pre_Builds\_Build_Scripts\auto_build_all.py --cuda_version cu118
|
||||
main(args)
|
||||
@@ -0,0 +1,33 @@
|
||||
repo_id: "MrForExample/Comfy3D_Pre_Builds"
|
||||
supported_cuda_versions: ["12.1", "11.8"]
|
||||
cuda_version: "12.1"
|
||||
|
||||
# Packages inside this file will be pip downloaded before actual build start
|
||||
build_requirements: "build_requirements.txt"
|
||||
# Dependencies to build wheels from
|
||||
dependencies: "dependencies.txt"
|
||||
# Folder used to store `git clone` project files from dependencies
|
||||
dependencies_dir_name: "_Build_Dependencies"
|
||||
# Folder used to store library files download from remote
|
||||
libs_dir_name: "_Libs"
|
||||
# Folder used to store built wheels
|
||||
wheels_dir_name: "_Build_Wheels"
|
||||
|
||||
build_base_packages: ["torch", "torchvision"]
|
||||
# All packages will be automatically installed at beginning of install.py follwing orders from top to bottom
|
||||
remote_packages:
|
||||
torch:
|
||||
version: "2.3.0"
|
||||
url: "https://download.pytorch.org/whl/${cuda_version}"
|
||||
|
||||
torchvision:
|
||||
version: "0.18.0"
|
||||
url: "https://download.pytorch.org/whl/${cuda_version}"
|
||||
|
||||
torch-scatter:
|
||||
url_option: "--find-links"
|
||||
url: "https://data.pyg.org/whl/torch-${remote_packages.torch.version}+${cuda_version}.html"
|
||||
|
||||
xformers:
|
||||
version: "0.0.26.post1"
|
||||
url: "https://download.pytorch.org/whl/${cuda_version}"
|
||||
@@ -1,2 +1,7 @@
|
||||
# build tools
|
||||
cmake
|
||||
ninja
|
||||
ninja
|
||||
|
||||
# web tools
|
||||
PyGithub
|
||||
requests
|
||||
@@ -0,0 +1,140 @@
|
||||
import sys
|
||||
import os
|
||||
from os.path import dirname
|
||||
import platform
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
PYTHON_PATH = sys.executable
|
||||
|
||||
try:
|
||||
from omegaconf import OmegaConf
|
||||
except ImportError as e:
|
||||
subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", ])
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
BUILD_SCRIPT_ROOT_ABS_PATH = dirname(os.path.abspath(__file__))
|
||||
build_config = OmegaConf.load(os.path.join(BUILD_SCRIPT_ROOT_ABS_PATH, "build_config.yaml"))
|
||||
|
||||
DEPENDENCIES_FILE_ABS_PATH = os.path.join(BUILD_SCRIPT_ROOT_ABS_PATH, build_config.dependencies)
|
||||
BUILD_REQUIREMENTS_FILE_ABS_PATH = os.path.join(BUILD_SCRIPT_ROOT_ABS_PATH, build_config.build_requirements)
|
||||
BUILD_ROOT_ABS_PATH = dirname(BUILD_SCRIPT_ROOT_ABS_PATH)
|
||||
DEPENDENCIES_ROOT_ABS_PATH = os.path.join(BUILD_ROOT_ABS_PATH, build_config.dependencies_dir_name)
|
||||
WHEELS_ROOT_ABS_PATH = os.path.join(BUILD_ROOT_ABS_PATH, build_config.wheels_dir_name)
|
||||
LIBS_ROOT_ABS_PATH = os.path.join(BUILD_ROOT_ABS_PATH, build_config.libs_dir_name)
|
||||
|
||||
def get_python_version():
|
||||
# Output: only first two version numbers, e.g. 3.12.4 -> py312
|
||||
return "py" + "".join(platform.python_version().split('.')[:-1])
|
||||
|
||||
def get_cuda_version():
|
||||
# Output: e.g. "cu121" or cu118
|
||||
result = subprocess.run(["nvcc", "--version"], shell=True, text=True, capture_output=True)
|
||||
if result.returncode == 0:
|
||||
for cuda_version in build_config.supported_cuda_versions:
|
||||
if "cuda_" + cuda_version in result.stdout:
|
||||
return "cu" + cuda_version.replace(".", "")
|
||||
|
||||
raise RuntimeError(f"Please install/reinstall CUDA tookit with any of the following supported version: {build_config.supported_cuda_versions}")
|
||||
|
||||
PYTHON_VERSION = get_python_version()
|
||||
CUDA_VERSION = get_cuda_version()
|
||||
build_config.cuda_version = CUDA_VERSION
|
||||
|
||||
def get_platform_config_name():
|
||||
platform_config_name = "_Wheels"
|
||||
|
||||
# Add OS Type
|
||||
if platform.system() == 'Windows':
|
||||
platform_config_name += "_win"
|
||||
elif platform.system() == "Linux":
|
||||
platform_config_name += "_linux"
|
||||
else:
|
||||
raise NotImplementedError(f"Platform {platform.system()} not supported!")
|
||||
|
||||
# Add Python Version
|
||||
platform_config_name += "_" + PYTHON_VERSION
|
||||
|
||||
# Add CUDA Version
|
||||
platform_config_name += "_" + CUDA_VERSION
|
||||
|
||||
return platform_config_name
|
||||
|
||||
def calculate_runtime(start_time):
|
||||
end_time = time.time()
|
||||
hours, rem = divmod(end_time - start_time, 3600)
|
||||
minutes, seconds = divmod(rem, 60)
|
||||
return hours, minutes, seconds
|
||||
|
||||
def git_file(data):
|
||||
try:
|
||||
import requests
|
||||
|
||||
c, out, folder = data
|
||||
|
||||
r = requests.get(c.download_url)
|
||||
output_path = c.path[len(folder):]
|
||||
output_abs_path = out + output_path
|
||||
os.makedirs(dirname(output_abs_path), exist_ok=True)
|
||||
with open(output_abs_path, 'wb') as f:
|
||||
print(f"Downloading {output_path} to {output_abs_path}")
|
||||
f.write(r.content)
|
||||
return False, c
|
||||
|
||||
except Exception as e:
|
||||
print(f"Exception in download_url(): {c.download_url}", e)
|
||||
return True, c
|
||||
|
||||
def git_contents_in_folder(repo, folder: str, recursive: bool=True):
|
||||
# Modified from https://github.com/Nordgaren/Github-Folder-Downloader
|
||||
contents = repo.get_contents(folder)
|
||||
file_contents = []
|
||||
for c in contents:
|
||||
if c.download_url is None:
|
||||
if recursive:
|
||||
file_contents += git_contents_in_folder(repo, c.path, recursive)
|
||||
continue
|
||||
|
||||
file_contents.append(c)
|
||||
|
||||
return file_contents
|
||||
|
||||
def git_folder_parallel(repo_id: str, folder: str, recursive: bool=True, root_outdir: str=""):
|
||||
try:
|
||||
from github import Github
|
||||
from multiprocessing import cpu_count
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Get all the file contents from repo (i.e. urls, relative path)
|
||||
repo = Github().get_repo(repo_id)
|
||||
file_contents = git_contents_in_folder(repo, folder, recursive)
|
||||
|
||||
inputs = zip(file_contents, [root_outdir] * len(file_contents), [folder] * len(file_contents))
|
||||
with ThreadPoolExecutor(max_workers=cpu_count() - 1) as executor:
|
||||
results = executor.map(git_file, inputs)
|
||||
#results = ThreadPool(cpus - 1).imap_unordered(git_file, inputs)
|
||||
for git_failed, c in results:
|
||||
if git_failed:
|
||||
raise RuntimeError(f"Could not download {c.path} from {c.download_url}, please check your internet connection.")
|
||||
|
||||
hours, minutes, seconds = calculate_runtime(start_time)
|
||||
print(f"Git folder finished in {int(hours):0>2}:{int(minutes):0>2}:{seconds:05.2f}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Couldn't download folder {folder} from repo {repo_id}", e)
|
||||
return False
|
||||
|
||||
def install_remote_packages(package_names):
|
||||
for package_name in package_names:
|
||||
package_attr = build_config.remote_packages[package_name]
|
||||
if hasattr(package_attr, "version"):
|
||||
package_name += f"=={package_attr.version}"
|
||||
url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url"
|
||||
|
||||
subprocess.run([
|
||||
PYTHON_PATH, "-s", "-m", "pip", "install",
|
||||
package_name, url_option, package_attr.url
|
||||
])
|
||||
Reference in New Issue
Block a user