diff --git a/CHANGELOG.md b/CHANGELOG.md index 13ab71f6..34608f5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -127,6 +127,31 @@ and this project adheres to [Semantic Versioning 2.0.0](https://semver.org/spec/ one bundled with the Manager is used. An empty result that an earlier version cached is replaced at the next start that reaches the channel, or goes away when the files in `/__manager/cache` are deleted (#3480). +- Judge a requirement for a `downgrade_blacklist` package by all of its clauses. + One that allows the installed version is installed when another clause sets a + minimum above it, as in `transformers==4.*,>=4.58.0` beside an installed + `4.57.0`, or excludes it with `!=` while a newer version is allowed, as in + `transformers~=4.57.0,!=4.57.0`; without extras both were skipped, so the + upgrade did not happen. One that the installed version does not meet and + that allows no newer version, as in `transformers[torch]<=4.57.0,!=4.57.0` or + `transformers<4.58,!=4.57.*`, is skipped; it was installed, and pip replaced + the package with an older version. A pre-release counts as a newer version + only when `==`, `>=`, `<=`, `~=` or `===` names one: pip before 25.0 takes no + other. +- At startup, read `>` as pip reads it: a post-release of the version, such as + `1.0.post1` for `>1.0`, does not meet it, and the requirement is installed. A + wildcard such as `==0.21.*` is met by an installed pre-release `0.21.0rc1`, + and the requirement is no longer passed to pip. +- At startup, install a requirement that excludes the installed version with + `!=` and allows a newer one, as in `numpy>=2.2.0,!=2.2.0` beside an installed + `2.2.0`, as an immediate install does; it was taken as installed already. pip + chooses the version, and that is an older one when no newer release exists. +- Leave an installed local build, such as the CUDA build `2.5.1+cu124`, alone + on all of these grounds. For a `downgrade_blacklist` package, keep it as well + when a requirement excludes it with `!=`, as in `xformers!=0.0.30` beside + `0.0.30+cu126`, unless the requirement sets a minimum above it; pip replaced + the build with whatever it found. For such a build, these changes only add + requirements that are skipped. ## [4.3] - 2026-09-19 diff --git a/comfyui_manager/common/manager_util.py b/comfyui_manager/common/manager_util.py index c3b4d3b5..11b3a0ae 100644 --- a/comfyui_manager/common/manager_util.py +++ b/comfyui_manager/common/manager_util.py @@ -18,7 +18,7 @@ import time from functools import lru_cache from packaging.requirements import InvalidRequirement, Requirement -from packaging.specifiers import InvalidSpecifier, Specifier +from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet from packaging.version import InvalidVersion, Version @@ -353,6 +353,72 @@ def _upper_bound_state(operator, version, installed): return 'above' if installed > bound else 'below' +def _is_met(clauses, installed): + """Whether the installed version meets every clause.""" + return all(Specifier(operator + version).contains(installed, prereleases=True) for operator, version in clauses) + + +def _is_excluded(clauses, installed): + """Whether a '!=' clause rejects the installed version.""" + return any(operator == '!=' and not Specifier('!=' + version).contains(installed, prereleases=True) + for operator, version in clauses) + + +def _newer_version_allowed(clauses, installed): + """Whether some version newer than the installed one meets every clause. + + Which versions exist is not known here. The clauses are asked about the versions + next to the installed one and next to the version of each clause, where a range + that a clause closes or excludes ends: the version itself, the one right after it, + and the next one at each level, which for '4.57.0' are '4.57.0.1', '4.57.1', '4.58' + and '5'; for a pre-release, a post-release or a development release, the next one + of its kind as well. Versions are written as long as the longest of them, so that + '1.0' beside '<1.0.1' is followed by '1.0.0.1'. The first post-release of a version + is asked about too. + + A pre-release is asked about only when '==', '>=', '<=', '~=' or '===' names one: + every pip takes a pre-release then. Otherwise pip may install an older release + instead; pip before 25.0 does so even when '<' or '>' names a pre-release. + + Another local build of the installed version is not asked about unless a clause + names it: a label says nothing about being newer, and pip would most often find + the build without a label. + """ + requirement = SpecifierSet(','.join(operator + version for operator, version in clauses)) + anchors = [installed] + pre_release_named = False + for operator, version in clauses: + try: + anchor = Version(version[:-2] if version.endswith('.*') else version) + except InvalidVersion: + continue # '===' may name anything + anchors.append(anchor) + if operator in ('==', '>=', '<=', '~=', '===') and anchor.is_prerelease: + pre_release_named = True + + width = max(len(anchor.release) for anchor in anchors) + candidates = [] + for anchor in anchors: + epoch = f'{anchor.epoch}!' if anchor.epoch else '' + padded = anchor.release + (0,) * (width - len(anchor.release)) + releases = [padded, padded + (1,)] + releases += [padded[:level] + (padded[level] + 1,) for level in range(width)] + candidates += [epoch + '.'.join(map(str, release)) for release in releases] + candidates.append(str(anchor)) + if anchor.pre is not None: + candidates.append(f'{anchor.base_version}{anchor.pre[0]}{anchor.pre[1] + 1}') + candidates.append(f'{anchor.base_version}.post{0 if anchor.post is None else anchor.post + 1}') + if anchor.dev is not None: + candidates.append(f"{anchor.public.rsplit('.dev', 1)[0]}.dev{anchor.dev + 1}") + + for candidate in map(Version, candidates): + if candidate.is_prerelease and not pre_release_named: + continue + if candidate > installed and requirement.contains(candidate, prereleases=True): + return True + return False + + def downgrade_protection(spec, clauses, extras, installed): """Decide whether a requirement may replace a downgrade-protected installed package. @@ -364,6 +430,18 @@ def downgrade_protection(spec, clauses, extras, installed): installed version, where pip would change nothing. With extras pip would still add the extras' dependencies, so those two cases are allowed. + The clauses are weighed together. A clause that allows the installed version does + not block the requirement when another clause sets a minimum above that version or + excludes it with '!=': pip then installs another version, and which one is left to + pip. When the installed version does not meet the clauses and no newer version meets + them either, the requirement is blocked, with or without extras, because pip has + only older versions to install. + + An installed local build, such as a CUDA build, is given up only for a minimum above + it. A requirement that allows it is blocked without extras, whatever its other + clauses, and one that excludes it with '!=' is blocked unless it sets such a minimum: + pip would replace the build with whatever it finds. + It is blocked as well when a clause cannot be compared with the installed version; `error` then says what was skipped, why, and how to proceed. """ @@ -379,32 +457,55 @@ def downgrade_protection(spec, clauses, extras, installed): if operator not in ('<', '<=', '==', '===', '~='): raise InvalidSpecifier(operator) states.add(_upper_bound_state(operator, version, current)) + only_older = not _is_met(clauses, current) and not _newer_version_allowed(clauses, current) + local_kept = (_is_excluded(clauses, current) and current.local is not None + and not requires_newer_version(clauses, installed)) except (InvalidSpecifier, InvalidVersion): return True, ( f"Skipped '{spec.strip()}': this package is protected by downgrade_blacklist, and its installed " f"version ({installed}) cannot be compared with the requested version. Install it manually if it is " "needed, or remove the package from downgrade_blacklist in config.ini if it was added there.") - return 'above' in states or ('at' in states and not extras), None + if 'above' in states or only_older or local_kept: + return True, None + if 'at' in states and not extras: + # requires_newer_version() is also true when '!=' excludes the installed version + return current.local is not None or not requires_newer_version(clauses, installed), None + return False, None def requires_newer_version(clauses, installed): - """Whether any clause sets a minimum that the installed version does not reach. + """Whether the installed version fails the requirement where a newer version can meet it. - Clauses that cannot be compared count as requiring it, so that pip decides. + That is so when a clause sets a minimum above the installed version. It is also so + when '!=' or '>' leaves out the installed version itself and a newer version meets + every clause. + + An installed local build, such as a CUDA build, is replaced only for a minimum above + it. '!=' is no ground for it, nor is '>' of its own version. + + A clause is read as pip reads it. '>' also leaves out a post-release and a local + build of its version, and '==4.*' is met by a pre-release of 4.0. Clauses that + cannot be compared count as requiring a newer version, so that pip decides. """ try: current = Version(installed) + left_out = False # by '>', although the installed version is not below its version for operator, version in clauses: if operator in ('>', '>=', '==', '===', '~='): + if Specifier(operator + version).contains(current, prereleases=True): + continue bound = Version(version[:-2] if version.endswith('.*') else version) - if current < bound or (operator == '>' and current == bound): + if current < bound: return True + left_out = left_out or operator == '>' elif operator in ('@', '?'): return True + if current.local is not None: + return False + return (left_out or _is_excluded(clauses, current)) and _newer_version_allowed(clauses, current) except InvalidVersion: return True - return False def pip_block_reason(spec, blacklist, downgrade_blacklist): diff --git a/tests/test_pip_blacklist_matching.py b/tests/test_pip_blacklist_matching.py index f50011a1..2821f69f 100644 --- a/tests/test_pip_blacklist_matching.py +++ b/tests/test_pip_blacklist_matching.py @@ -14,6 +14,7 @@ from pathlib import Path from types import SimpleNamespace import pytest +from packaging.specifiers import Specifier from _install_flags_testutil import import_context, import_reader @@ -22,9 +23,12 @@ from _install_flags_testutil import import_context, import_reader PIP_LIST = '''\ Package Version --------------- ------------ +einops 0.8.0.post1 numpy 2.2.0 opencv-python 5.0.0.93 ruamel.yaml 0.18.6 +safetensors 0.5.0.dev1 +tokenizers 0.21.0rc1 transformers 4.57.0 triton-windows 3.5.0 xformers 0.0.30+cu126 @@ -107,6 +111,14 @@ def test_dots_are_name_separators(set_lists, is_blocked, entry, requirement): 'transformers==4.0.0 --index-url https://example.invalid/simple', 'opencv-python<5', # 5.0.0.93 is newer than 5 'xformers==0.0.20', # installed with a local version label + # The installed version is excluded and nothing newer is allowed, so pip has only older versions. + 'transformers<=4.57.0,!=4.57.0', + 'transformers[torch]<=4.57.0,!=4.57.0', + 'transformers[torch]!=4.57.*,<=4.57.0', + 'transformers[torch]>=4.50,<=4.57.0,!=4.57.0', + 'transformers<4.58,!=4.57.*', # the excluded range reaches the upper bound + 'transformers[torch]<4.58,!=4.57.*', + 'transformers[torch]~=4.57.0,!=4.57.*', ]) def test_downgrade_blacklist_blocks_possible_downgrades(set_lists, is_blocked, caplog, requirement): set_lists(downgrade_blacklist=['transformers', 'triton-windows', 'opencv-python', 'xformers']) @@ -136,6 +148,20 @@ def test_downgrade_blacklist_blocks_possible_downgrades(set_lists, is_blocked, c 'transformers[sentencepiece]==4.57.0', 'transformers[sentencepiece]~=4.57.0', 'transformers[sentencepiece]>=4.0,<5', + # A clause allows the installed version, but another sets a minimum above it: an upgrade. + 'transformers==4.*,>=4.58.0', + 'transformers~=4.57.0,>4.57.0', + # The installed version is excluded, but newer versions are allowed: which version is left to pip. + 'transformers!=4.57.0', + 'transformers~=4.57.0,!=4.57.0', + 'transformers==4.*,!=4.57.0', + 'transformers[torch]==4.*,!=4.57.0', + 'transformers[torch]<5,!=4.57.0', + 'transformers[torch]~=4.57.0,!=4.57.0', + 'transformers<4.57.1,!=4.57.0', # a post-release of 4.57.0 is left + 'transformers<4.57.0.1,!=4.57', # 4.57.0.0.1 is left: versions are written at one length + 'transformers<4.59,!=4.57.*', # 4.58 is left + 'transformers!=4.57.*,!=4.58.*,<5', # 4.59 is left ]) def test_downgrade_blacklist_allows_other_requirements(set_lists, is_blacklisted, caplog, requirement): set_lists(downgrade_blacklist=['transformers']) @@ -217,8 +243,44 @@ def test_startup_check_reports_downgrade_protection(set_lists, capsys): ('opencv-python>=6', False), ('opencv-python~=5.1', False), # older than the compatible release ('opencv-python~=5.0.0', True), + ('numpy>2.2.0', False), # `>` excludes the installed version + ('numpy>2.1', True), + ('numpy>2.2.0,<2.2.1', False), # 2.2.0.1 is left; `>` leaves out the post-releases of 2.2.0 + ('numpy>2.2,<2.2.0.1', False), # 2.2.0.0.1 is left: versions are written at one length + ('numpy>2.2.0,<=2.2.0', True), # nothing newer is left ('ruamel-yaml>=0.17', True), # listed by pip as ruamel.yaml ('xformers>=0.0.20', True), # installed with a local version label + ('xformers>=0.0.30', True), + ('xformers>0.0.30', True), # an installed local build is left alone + ('xformers!=0.0.30', True), + ('xformers~=0.0.30,!=0.0.30+cu126', True), + ('xformers>=0.0.31', False), # a minimum above it is still installed + ('einops>=0.8.0', True), # installed as a post-release + ('einops>0.8.0', False), # `>` also excludes a post-release of that version + ('einops>0.7', True), + ('tokenizers==0.21.*', True), # installed as a pre-release, which the wildcard includes + ('tokenizers>=0.21.0rc1', True), + ('tokenizers>=0.21.0', False), # a pre-release is older than its release + ('tokenizers==0.21.0', False), + # `!=` excludes the installed version: pip installs another one, unless only older ones are left. + ('numpy!=2.2.0', False), + ('numpy>=2.2.0,!=2.2.0', False), + ('numpy~=2.2.0,!=2.2.0', False), + ('numpy==2.*,!=2.2.*', False), + ('numpy<3,!=2.2.0', False), + ('numpy!=2.1.0', True), # another version is excluded + ('numpy<=2.2.0,!=2.2.0', True), # nothing newer is allowed + ('numpy==2.2.0,!=2.2.0', True), + ('numpy<2.2.0,!=2.2.0', True), # the upper bound is below the installed version + ('numpy<2.3,!=2.2.*', True), # the excluded range reaches the upper bound + ('numpy<2.4,!=2.2.*', False), + ('einops<0.8.0.post3,!=0.8.0.post1', False), # the next post-release is left + ('numpy<2.2.0.post2,!=2.2.0', False), # a post-release of 2.2.0 is left + ('tokenizers<=0.21.0rc2,!=0.21.0rc1', False), # the next pre-release is left, and `<=` names one + ('tokenizers<0.21.0rc3,!=0.21.0rc1', True), # only `<` names one: pip before 25.0 would install an older release + ('safetensors>=0.5.0.dev1,!=0.5.0.dev1,<0.5.0.dev3', False), # the next development release is left + ('tokenizers<=0.21.0,!=0.21.0,!=0.21.0rc1', True), # only a pre-release is left, which pip would not take + ('einops>0.8.0rc1,<=0.8.0.post1', True), # `>` leaves out 0.8.0.post1, and only 0.8.0 is left ('opencv-python[extra]', False), # extras to add: pip decides ('opencv-python @ https://example.invalid/opencv.whl', False), # no version: pip decides ]) @@ -227,6 +289,67 @@ def test_startup_check_for_packages_on_neither_list(set_lists, requirement, skip assert load_check('prestartup')(requirement) is skipped +def test_an_installed_local_build_is_not_replaced_on_the_new_grounds(set_lists, is_blocked): + """A local build, such as a CUDA build, is given up only for a minimum above it or a build that the requirement names.""" + set_lists(downgrade_blacklist=['xformers']) + + # pip would most often find the build without a label, which is older than the installed one + assert is_blocked('xformers==0.0.30,!=0.0.30+cu126') is True + assert is_blocked('xformers[extra]<=0.0.30,!=0.0.30+cu126') is True + assert is_blocked('xformers~=0.0.30,!=0.0.30') is True + assert is_blocked('xformers[extra]~=0.0.30,!=0.0.30') is True + assert is_blocked('xformers==0.*,>=0.0.31') is True + # `!=` alone leaves the choice to pip, which may replace the build with any other + assert is_blocked('xformers!=0.0.30') is True + assert is_blocked('xformers<1,!=0.0.30+cu126') is True + + assert is_blocked('xformers>=0.0.31') is False + assert is_blocked('xformers>=0.0.31,!=0.0.30') is False + assert is_blocked('xformers==0.0.30+cu128,!=0.0.30+cu126') is False + + +def test_a_requirement_that_leaves_only_an_older_version_to_pip_is_blocked(set_lists, is_blocked): + """What counts is the version pip would install, read as pip reads the requirement.""" + set_lists(downgrade_blacklist=['tokenizers', 'einops', 'safetensors']) + + # pip takes a pre-release only when the requirement names one; here it would take an older release + assert is_blocked('tokenizers<=0.21.0,!=0.21.0,!=0.21.0rc1') is True + assert is_blocked('tokenizers<=0.21.0rc2,!=0.21.0rc1') is False + assert is_blocked('tokenizers>=0.21.0rc1,!=0.21.0rc1,<0.21.0rc3') is False # 0.21.0rc2 is left + assert is_blocked('safetensors>=0.5.0.dev1,!=0.5.0.dev1,<0.5.0.dev3') is False # 0.5.0.dev2 is left + # pip before 25.0 does not take one that only `<` or `>` names + assert is_blocked('tokenizers<0.21.0rc3,!=0.21.0rc1') is True + assert is_blocked('safetensors!=0.5.0.dev1,<0.5.0.dev3') is True + + # `>0.8.0` leaves out the installed 0.8.0.post1, and no version is left + assert is_blocked('einops>0.8.0,<=0.8.0.post1') is True + assert is_blocked('einops[extra]>0.8.0,<=0.8.0.post1') is True + assert is_blocked('einops>0.8.0,<=0.8.1') is False + + +def test_a_post_release_that_greater_than_leaves_out_is_not_replaced_by_its_release(set_lists, is_blocked, is_blacklisted): + """packaging before 26.1 reads `>0.8.0rc1` as leaving out 0.8.0.post1 while allowing 0.8.0, which is older.""" + set_lists(downgrade_blacklist=['einops']) + left_out = not Specifier('>0.8.0rc1').contains('0.8.0.post1', prereleases=True) + + assert is_blocked('einops>0.8.0rc1,<=0.8.0.post1') is True # left out or not, nothing newer is allowed + # Where it is not left out, the requirement is met: pip keeps the version and adds the extras. + assert is_blocked('einops[extra]>0.8.0rc1,<=0.8.0.post1') is left_out + assert is_blocked('einops[extra]>0.8.0rc1,<=0.8.0.post2') is left_out # the upper bound is above the installed version + assert is_blacklisted('einops>0.8.0rc1,<=0.8.0.post2') is left_out + assert is_blacklisted('einops>0.8.0rc1,<0.9') is False # 0.8.1 is allowed + + +def test_startup_check_agrees_with_downgrade_protection_on_an_excluded_version(set_lists): + """A protected package whose installed version is excluded is installed at startup as it is immediately.""" + set_lists(downgrade_blacklist=['transformers']) + is_installed = load_check('prestartup') + + assert is_installed('transformers~=4.57.0,!=4.57.0') is False + assert is_installed('transformers!=4.57.0') is False + assert is_installed('transformers<=4.57.0,!=4.57.0') is True # only older versions are left: skipped + + REQUIREMENTS_TXT = '''\ torch --index-url https://example.invalid/simple opencv-python~=4.8.1.78 # pinned by the pack