Files
Comfy-Org-ComfyUI-Manager/tests/test_pygit2_submodules.py
T
Dr.Lt.Data fdcd6122e2 v4: Fix git_exe, start-up without git, .git/config reading, submodules and the git steps of install and update
Five fixes. The CHANGELOG has an entry for each.

- Start with a node pack whose .git/config cannot be read
- Use the git executable set as git_exe
- Read the commit of a nodepack whose refs are packed, and bring its
  submodules to the commit that is checked out
- Update nested submodules with pygit2, fetch them through the proxy and
  over https, and fail when submodules could not be updated
- Do not end a clone or a fetch of pygit2 at a progress message of the
  server that is not UTF-8 by itself
- Fail an update whose pull or submodule update failed, leave the nodepack
  at its commit, remove what a clone that failed left, and refuse an install
  of a nodepack that another install is working on
2026-11-18 00:10:04 +09:00

339 lines
16 KiB
Python

"""The pygit2 backend updates the submodules of a repository as `git submodule update --init --recursive` does.
Each case runs the update in a process of its own, with pygit2 asked for and no git executable on `PATH`.
"""
import os
import shutil
import socket
import ssl
import subprocess
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import urlsplit
import pytest
from _install_flags_testutil import GIT_LOCATION
from test_git_commit_and_submodules import git_with_local_submodules, head, repository, repository_with_a_submodule
pytest.importorskip('pygit2')
# For a git server: what `pack-objects` writes to stderr reaches the client as a progress message, here the first
# two bytes of a Korean character. pygit2 decodes each piece of a message, and a piece can end like that.
MESSAGE_CUT_IN_A_CHARACTER = {'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'uploadpack.packObjectsHook',
'GIT_CONFIG_VALUE_0': """sh -c 'printf "\\355\\225" >&2; exec "$@"' --"""}
@pytest.fixture(autouse=True)
def git_finds_its_repository_by_the_directory(monkeypatch):
for name in GIT_LOCATION:
monkeypatch.delenv(name, raising=False)
def run_with_pygit2(tmp_path, code):
"""In a process of its own, with no git executable to fall back on."""
(tmp_path / 'no-git').mkdir(exist_ok=True)
common = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'comfyui_manager', 'common')
code = f'import sys; sys.path.insert(0, {common!r}); import git_compat\nassert git_compat.USE_PYGIT2\n{code}\n'
env = {key: value for key, value in os.environ.items() if key not in ('GIT_EXE_PATH', 'GIT_PYTHON_GIT_EXECUTABLE')}
return subprocess.run([sys.executable, '-c', code], env={**env, 'PATH': str(tmp_path / 'no-git'), 'CM_USE_PYGIT2': '1'},
capture_output=True, text=True)
def update_submodules_with_pygit2(tmp_path, clone, init=True, first=''):
return run_with_pygit2(tmp_path, f'{first}\ngit_compat.open_repo({str(clone)!r}).submodule_update(init={init})')
def test_pygit2_updates_the_fetched_submodules_beside_one_never_fetched(tmp_path):
"""Without a git executable there is no fallback: pygit2 refuses a submodule that was never fetched."""
library = repository(tmp_path / 'library')
origin = tmp_path / 'origin'
origin.mkdir()
git_with_local_submodules(origin, 'init', '-q')
for name in ('a-never-fetched', 'b-fetched'):
git_with_local_submodules(origin, 'submodule', 'add', '-q', str(tmp_path / 'library'), name)
git_with_local_submodules(origin, 'commit', '-q', '-a', '-m', 'two submodules')
clone = tmp_path / 'clone'
git_with_local_submodules(tmp_path, 'clone', '-q', str(origin), str(clone))
git_with_local_submodules(clone, 'submodule', 'update', '-q', '--init', 'b-fetched')
git_with_local_submodules(clone / 'b-fetched', 'checkout', '-q', library[0])
(tmp_path / 'no-git').mkdir()
common = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'comfyui_manager', 'common')
code = (f'import sys; sys.path.insert(0, {common!r}); import git_compat\n'
f'assert git_compat.USE_PYGIT2\ngit_compat.open_repo({str(clone)!r}).submodule_update(init=False)\n')
env = {key: value for key, value in os.environ.items() if key not in ('GIT_EXE_PATH', 'GIT_PYTHON_GIT_EXECUTABLE')}
child = subprocess.run([sys.executable, '-c', code], env={**env, 'PATH': str(tmp_path / 'no-git'), 'CM_USE_PYGIT2': '1'},
capture_output=True, text=True)
assert child.returncode == 0, child.stderr
assert head(clone / 'b-fetched') == library[1]
assert os.listdir(clone / 'a-never-fetched') == []
def test_pygit2_updates_a_submodule_of_a_submodule(tmp_path):
"""As `git submodule update --init --recursive` does, which the other backend runs."""
library, _ = repository_with_a_submodule(tmp_path)
outer = tmp_path / 'outer'
outer.mkdir()
git_with_local_submodules(outer, 'init', '-q')
git_with_local_submodules(outer, 'submodule', 'add', '-q', str(tmp_path / 'origin'), 'origin')
git_with_local_submodules(outer, 'commit', '-q', '-a', '-m', 'a submodule with a submodule')
clone = tmp_path / 'clone'
git_with_local_submodules(tmp_path, 'clone', '-q', str(outer), str(clone))
child = update_submodules_with_pygit2(tmp_path, clone)
assert child.returncode == 0, child.stderr
assert (clone / 'origin/library/__init__.py').exists()
assert head(clone / 'origin/library') == library[1]
git_with_local_submodules(clone / 'origin/library', 'checkout', '-q', library[0])
child = update_submodules_with_pygit2(tmp_path, clone)
assert child.returncode == 0, child.stderr
assert head(clone / 'origin/library') == library[1]
# without `init` as well, which a restore asks for when the nodepack is at its commit already
git_with_local_submodules(clone / 'origin/library', 'checkout', '-q', library[0])
child = update_submodules_with_pygit2(tmp_path, clone, init=False)
assert child.returncode == 0, child.stderr
assert head(clone / 'origin/library') == library[1]
def test_pygit2_passes_over_a_submodule_that_gitmodules_alone_names(tmp_path):
"""As git does: an entry left in `.gitmodules` for a submodule that was removed is not a failure."""
library, _ = repository_with_a_submodule(tmp_path)
with open(tmp_path / 'origin/.gitmodules', 'a', encoding='utf-8') as gitmodules:
gitmodules.write(f'[submodule "a-removed"]\n\tpath = a-removed\n\turl = {tmp_path / "nowhere"}\n')
git_with_local_submodules(tmp_path / 'origin', 'commit', '-q', '-a', '-m', 'an entry without a submodule')
clone = tmp_path / 'clone'
git_with_local_submodules(tmp_path, 'clone', '-q', str(tmp_path / 'origin'), str(clone))
child = update_submodules_with_pygit2(tmp_path, clone)
assert child.returncode == 0, child.stderr
assert head(clone / 'library') == library[1]
def add_repository_without_gitmodules(tmp_path, origin):
"""A repository committed inside `origin` by `git add`: a commit in the tree that `.gitmodules` does not name."""
repository(origin / 'stray')
git_with_local_submodules(origin, 'add', 'stray')
git_with_local_submodules(origin, 'commit', '-q', '-m', 'a repository added without `git submodule`')
clone = tmp_path / 'clone'
git_with_local_submodules(tmp_path, 'clone', '-q', str(origin), str(clone))
return clone
def test_pygit2_passes_over_a_commit_in_the_tree_without_gitmodules(tmp_path):
"""As `git clone --recursive` does: there is no url to fetch it from, and the nodepack is installed."""
repository(tmp_path / 'origin')
clone = add_repository_without_gitmodules(tmp_path, tmp_path / 'origin')
child = update_submodules_with_pygit2(tmp_path, clone)
assert child.returncode == 0, child.stderr
assert os.listdir(clone / 'stray') == []
def test_pygit2_updates_the_submodules_beside_a_commit_that_gitmodules_does_not_name(tmp_path):
library, _ = repository_with_a_submodule(tmp_path)
clone = add_repository_without_gitmodules(tmp_path, tmp_path / 'origin')
child = update_submodules_with_pygit2(tmp_path, clone)
assert child.returncode == 0, child.stderr
assert head(clone / 'library') == library[1]
def test_pygit2_does_not_pass_over_a_submodule_it_could_not_fetch(tmp_path):
"""Without a git executable nothing else does the update: the caller must not take it as done."""
repository_with_a_submodule(tmp_path)
clone = tmp_path / 'clone'
git_with_local_submodules(tmp_path, 'clone', '-q', str(tmp_path / 'origin'), str(clone))
shutil.rmtree(tmp_path / 'library')
child = update_submodules_with_pygit2(tmp_path, clone)
assert child.returncode != 0, child.stderr
assert 'GitCommandError' in child.stderr
def test_pygit2_does_not_pass_over_a_git_that_did_not_come_back(tmp_path):
repository_with_a_submodule(tmp_path)
clone = tmp_path / 'clone'
git_with_local_submodules(tmp_path, 'clone', '-q', str(tmp_path / 'origin'), str(clone))
shutil.rmtree(tmp_path / 'library')
late = ('import subprocess\n'
'def late(command, timeout, **_):\n'
' raise subprocess.TimeoutExpired(command, timeout)\n'
'subprocess.run = late')
child = update_submodules_with_pygit2(tmp_path, clone, first=late)
assert child.returncode != 0, child.stderr
assert 'TimeoutExpired' in child.stderr
def test_pygit2_fetches_a_submodule_whose_fetch_failed_at_the_update_before(tmp_path):
"""An update whose submodules cannot be fetched is undone; what the clone of the submodule left is not in the way of the next."""
repository(tmp_path / 'library')
origin, pack = tmp_path / 'origin', tmp_path / 'pack'
repository(origin)
git_with_local_submodules(tmp_path, 'clone', '-q', str(origin), str(pack))
git_with_local_submodules(origin, 'submodule', 'add', '-q', str(tmp_path / 'library'), 'library')
git_with_local_submodules(origin, 'commit', '-q', '-m', 'a submodule')
(tmp_path / 'library').rename(tmp_path / 'library.away')
# as an update of Manager does it
update = (f'repo = git_compat.open_repo({str(pack)!r})\n'
'commit = repo.head_commit_hexsha\n'
'repo.pull_ff_only()\n'
'try:\n'
' repo.submodule_update()\n'
'except Exception:\n'
' repo.reset_hard(commit)\n'
' raise')
child = run_with_pygit2(tmp_path, update)
assert child.returncode != 0, child.stderr
assert head(pack) != head(origin)
(tmp_path / 'library.away').rename(tmp_path / 'library')
child = run_with_pygit2(tmp_path, update)
assert child.returncode == 0, child.stderr
assert head(pack) == head(origin)
assert (pack / 'library/__init__.py').read_text() == 'VERSION = 2'
class GitHostBehindAProxy(BaseHTTPRequestHandler):
"""A proxy, and behind it the https host of the repositories of `server.root`: the tunnel it is asked for ends in itself."""
protocol_version = 'HTTP/1.1'
def handle(self):
# one that did not ask the proxy begins with a TLS hello, and is not answered
if self.connection.recv(1, socket.MSG_PEEK) == b'C':
super().handle()
def do_CONNECT(self):
self.send_response(200)
self.end_headers()
self.connection = self.server.tls.wrap_socket(self.connection, server_side=True)
self.rfile = self.connection.makefile('rb')
self.wfile = self.connection.makefile('wb', 0)
def finish(self):
super().finish()
self.connection.close()
def do_GET(self):
url = urlsplit(self.path)
self.server.asked.add(url.path.split('/')[1])
body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
cgi = {'GIT_PROJECT_ROOT': self.server.root, 'GIT_HTTP_EXPORT_ALL': '1', 'REQUEST_METHOD': self.command,
'PATH_INFO': url.path, 'QUERY_STRING': url.query, 'CONTENT_TYPE': self.headers.get('Content-Type', '')}
answer = subprocess.run([self.server.backend], input=body, env={**os.environ, **cgi, **MESSAGE_CUT_IN_A_CHARACTER}, check=True, capture_output=True).stdout
headers, _, content = answer.partition(b'\r\n\r\n')
status = headers.partition(b'Status: ')[2].split(b'\r\n')[0] or b'200 OK'
self.wfile.write(b'HTTP/1.1 %s\r\nContent-Length: %d\r\n%s\r\n\r\n%s' % (status, len(content), headers, content))
do_POST = do_GET
def log_message(self, *args):
pass
@pytest.fixture
def git_host_behind_a_proxy(tmp_path, monkeypatch):
"""The repositories of `tmp_path` at an https `url` that only the proxy of the global git configuration leads to."""
backend = os.path.join(subprocess.run(['git', '--exec-path'], check=True, capture_output=True, text=True).stdout.strip(), 'git-http-backend')
if not os.path.exists(backend) or shutil.which('openssl') is None:
pytest.skip('git-http-backend and openssl are needed')
# https, and a name: libgit2 leaves the proxy aside for http, and does not match a certificate with an address
certificate, key = str(tmp_path / 'certificate.pem'), str(tmp_path / 'key.pem')
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '2', '-keyout', key, '-out', certificate,
'-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], check=True, capture_output=True)
server = ThreadingHTTPServer(('127.0.0.1', 0), GitHostBehindAProxy)
server.tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
server.tls.load_cert_chain(certificate, key)
server.backend, server.root, server.asked = backend, str(tmp_path), set()
server.url = 'https://localhost:%d/' % server.server_port
# what the update reads in its own process: the proxy as git_compat finds it, and the certificate to trust
(tmp_path / 'home').mkdir()
(tmp_path / 'home/.gitconfig').write_text('[http]\n\tproxy = http://127.0.0.1:%d\n' % server.server_port)
monkeypatch.setenv('HOME', str(tmp_path / 'home'))
monkeypatch.setenv('SSL_CERT_FILE', certificate)
threading.Thread(target=server.serve_forever, daemon=True).start()
yield server
server.shutdown()
server.server_close()
def pin(path, submodule, commit):
"""Commits `submodule` at `commit` in the repository at `path`, which need not have that commit; the hash."""
git_with_local_submodules(path, 'update-index', '--cacheinfo', f'160000,{commit},{submodule}')
git_with_local_submodules(path, 'commit', '-q', '-m', f'{submodule} at {commit}')
return head(path)
def test_pygit2_fetches_the_submodules_through_the_proxy(tmp_path, git_host_behind_a_proxy):
"""The nodepack is cloned through `http.proxy` of the global git configuration: so are its submodules, and theirs."""
library, _ = repository_with_a_submodule(tmp_path)
origin, outer, clone = tmp_path / 'origin', tmp_path / 'outer', tmp_path / 'clone'
# relative: beside the url that its repository is cloned from
git_with_local_submodules(origin, 'config', '-f', '.gitmodules', 'submodule.library.url', '../library')
git_with_local_submodules(origin, 'commit', '-q', '-a', '-m', 'the library beside it')
outer.mkdir()
git_with_local_submodules(outer, 'init', '-q')
git_with_local_submodules(outer, 'submodule', 'add', '-q', '../origin', 'origin')
git_with_local_submodules(outer, 'commit', '-q', '-a', '-m', 'a submodule with a submodule')
cloned = f'git_compat.clone_repo({git_host_behind_a_proxy.url + "outer"!r}, {str(clone)!r})'
child = update_submodules_with_pygit2(tmp_path, clone, first=cloned)
assert child.returncode == 0, child.stderr
assert head(clone / 'origin/library') == library[1]
assert git_host_behind_a_proxy.asked == {'outer', 'origin', 'library'}
# commits that the clones of the submodules do not have yet
(tmp_path / 'library/__init__.py').write_text('VERSION = 3')
git_with_local_submodules(tmp_path / 'library', 'commit', '-q', '-a', '-m', 'version 3')
later = head(tmp_path / 'library')
pin(clone, 'origin', pin(origin, 'library', later))
child = update_submodules_with_pygit2(tmp_path, clone)
assert child.returncode == 0, child.stderr
assert head(clone / 'origin/library') == later
# and without `init`, which a restore asks for when the nodepack is at its commit already
(tmp_path / 'library/__init__.py').write_text('VERSION = 4')
git_with_local_submodules(tmp_path / 'library', 'commit', '-q', '-a', '-m', 'version 4')
latest = head(tmp_path / 'library')
pin(clone, 'origin', pin(origin, 'library', latest))
child = update_submodules_with_pygit2(tmp_path, clone, init=False)
assert child.returncode == 0, child.stderr
assert head(clone / 'origin/library') == latest
def test_pygit2_fetches_a_submodule_with_an_ssh_url_over_https(tmp_path, git_host_behind_a_proxy):
"""As the nodepack itself, whose ssh url is cloned over https: a public repository needs no credentials."""
library, _ = repository_with_a_submodule(tmp_path)
origin, clone = tmp_path / 'origin', tmp_path / 'clone'
git_with_local_submodules(origin, 'config', '-f', '.gitmodules', 'submodule.library.url', 'git@localhost:library')
git_with_local_submodules(origin, 'commit', '-q', '-a', '-m', 'an ssh url')
cloned = f'git_compat.clone_repo({git_host_behind_a_proxy.url + "origin"!r}, {str(clone)!r})'
child = update_submodules_with_pygit2(tmp_path, clone, first=cloned)
assert child.returncode == 0, child.stderr
assert head(clone / 'library') == library[1]