Make native source fingerprints portable across Python versions
Serialize every semantic AST field explicitly so identical reviewed Torch definitions produce the same fingerprint on Python 3.11 and 3.13. Retain loaded-code, decorator, closure and context-factory validation, with no new admitted source profile. Add regression coverage for empty fields, generic parameters, semantic edits and dependency rejection. All 185 tests and required compile checks pass; seven reference fingerprints and native decorator fixtures agree across both Python versions. Serving-host acceptance remains pending.
This commit is contained in:
@@ -277,6 +277,13 @@ against selected definitions from
|
||||
and [grad mode](https://github.com/pytorch/pytorch/blob/v2.9.0/torch/autograd/grad_mode.py).
|
||||
Identical selected definitions in another release satisfy the same function
|
||||
pins; unknown changes reject. `__wrapped__` metadata alone never grants admission.
|
||||
The source fingerprint uses a versioned explicit node/field/value representation,
|
||||
not `ast.dump` display formatting, which differs between Python3.11 and3.13.
|
||||
Empty semantic fields are retained. Nongeneric definitions on older Python get
|
||||
the same empty `type_params` field as newer Python; nonempty generic parameters,
|
||||
body, defaults, annotations and decorators all remain binding. This changes the
|
||||
fingerprint representation of the same reviewed definitions, not the admitted
|
||||
Torch semantics. Loaded-code, closure and context-factory checks still apply.
|
||||
Extra transparent or numeric-changing wrappers, altered factories, multiple
|
||||
layers and compiled wrappers remain unsupported. No PyTorch runtime code is
|
||||
bundled by this integration.
|
||||
|
||||
@@ -7,22 +7,48 @@ verification. No upstream implementation is copied or executed by compilation.
|
||||
import ast
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
from pathlib import Path
|
||||
import types
|
||||
|
||||
_TORCH_CONTEXT_AST = {
|
||||
'context_decorator': '967c7c22412aef1d6b4b82c59bb2485ecff8bdafe845adf4edb294141b61f752',
|
||||
'_DecoratorContextManager.__call__': '3dced21e5ccb84912cca3b933c2144d2e0e2432f2c68d9e03a0ad28873de3719',
|
||||
'_DecoratorContextManager.clone': '1b8edbd9e45faf2702a9a40c4003b10dd58a1ea9992681cef9ef8282a7ca8323',
|
||||
'_NoParamDecoratorContextManager.__new__': 'c4a90e1fd69958e3e3d629a76bad5970a2411d90ac1d5528c3a395de27986a6b',
|
||||
'context_decorator': 'ab88233fefe874378a063c5ba94e83e3c367c3d0f3a10b2285c36d20ee26b517',
|
||||
'_DecoratorContextManager.__call__': 'fd1b4f0e58ad7340d39e20dd2ffc37c887588baf07ae7b6f5ab4ac47b191d666',
|
||||
'_DecoratorContextManager.clone': 'df22dd330c061c2e3abb49ba6f554bc082f35cde53bc098d268d73f35a8c234a',
|
||||
'_NoParamDecoratorContextManager.__new__': '65c0570159bd596a54a15b9797c4465a2e093a3f0eb5f5bd80c3d9ba87f1023b',
|
||||
}
|
||||
_TORCH_GRAD_AST = {
|
||||
'no_grad.__init__': 'c1ec080424ff4617104126eebbf0b1333cc16a59c4dafbc408135e634103617e',
|
||||
'no_grad.__enter__': '3b78db38d86b55bd581f39994a9ae49a5d617039e4c2b754860421a25837161f',
|
||||
'no_grad.__exit__': '2a3cf08e8b1266c73995b8ffa1469a43b3fb34c54e770165b64c73b7dfd70ceb',
|
||||
'no_grad.__init__': '895fe28d68080b651f4918f4540adf6c15a9a98c28f39724468cb57749333c0b',
|
||||
'no_grad.__enter__': '6da10b3038d3031ca8b09fa5b72844b9cd271b9174e13731b3fb8b65779e7122',
|
||||
'no_grad.__exit__': 'cdb682833c9504854cd368e7fc3a29d7c46164136d4af9c84492aed1bc87846d',
|
||||
}
|
||||
|
||||
|
||||
def _ast_structure(value):
|
||||
"""Keep semantic fields explicit; avoid version-dependent ast.dump formatting."""
|
||||
if isinstance(value, ast.AST):
|
||||
fields = dict(ast.iter_fields(value))
|
||||
if isinstance(value, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
# Python3.11 lacks this field. It means no generic parameters, just
|
||||
# like an empty list in3.12+. Nonempty parameters remain binding.
|
||||
fields.setdefault('type_params', getattr(value, 'type_params', []))
|
||||
return [type(value).__name__, [[name, _ast_structure(fields[name])] for name in sorted(fields)]]
|
||||
if isinstance(value, list):
|
||||
return [_ast_structure(item) for item in value]
|
||||
if isinstance(value, bytes):
|
||||
return ['bytes', value.hex()]
|
||||
if isinstance(value, complex):
|
||||
return ['complex', value.real, value.imag]
|
||||
if value is Ellipsis:
|
||||
return ['ellipsis']
|
||||
return value
|
||||
|
||||
|
||||
def _ast_fingerprint(node):
|
||||
data = json.dumps(_ast_structure(node), ensure_ascii=True, separators=(',', ':')).encode('ascii')
|
||||
return hashlib.sha256(b'v12-ast-fields-v1\0' + data).hexdigest()
|
||||
|
||||
|
||||
def _constant(value):
|
||||
if isinstance(value, types.CodeType):
|
||||
return ('code', _code_key(value))
|
||||
@@ -81,7 +107,7 @@ def _torch_source(module, pins):
|
||||
codes, definitions = _source_maps(source)
|
||||
for name, expected_hash in pins.items():
|
||||
node = definitions.get(name)
|
||||
if node is None or hashlib.sha256(ast.dump(node, include_attributes=False).encode()).hexdigest() != expected_hash:
|
||||
if node is None or _ast_fingerprint(node) != expected_hash:
|
||||
raise RuntimeError('V12 native no_grad dependency source is unverified: ' + name)
|
||||
function = _lookup(module, name)
|
||||
_plain_code(function, codes.get(name), name)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Cross-version AST representation and semantic-change guard regressions."""
|
||||
import ast
|
||||
import copy
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from types import ModuleType
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SPEC = importlib.util.spec_from_file_location('ast_fingerprint_v12_tests', ROOT / 'kg_krea_v12/__init__.py',
|
||||
submodule_search_locations=[str(ROOT / 'kg_krea_v12')])
|
||||
PACKAGE = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = PACKAGE
|
||||
SPEC.loader.exec_module(PACKAGE)
|
||||
from ast_fingerprint_v12_tests import compatibility
|
||||
|
||||
|
||||
class AstFingerprintTests(unittest.TestCase):
|
||||
def test_absent_old_python_type_params_equals_explicit_empty_new_field(self):
|
||||
node = ast.parse('def step(x):\n return x + 1\n').body[0]
|
||||
old, new = copy.deepcopy(node), copy.deepcopy(node)
|
||||
if hasattr(old, 'type_params'):
|
||||
del old.type_params
|
||||
new.type_params = []
|
||||
self.assertEqual(compatibility._ast_fingerprint(old), compatibility._ast_fingerprint(new))
|
||||
new.type_params = [ast.Name(id='T', ctx=ast.Load())]
|
||||
self.assertNotEqual(compatibility._ast_fingerprint(old), compatibility._ast_fingerprint(new))
|
||||
|
||||
def test_body_defaults_annotations_and_decorators_remain_binding(self):
|
||||
reference = ast.parse('def step(x: int, *, enabled=True):\n return x + 1\n').body[0]
|
||||
original = compatibility._ast_fingerprint(reference)
|
||||
variants = [
|
||||
'def step(x: int, *, enabled=True):\n return x + 2\n',
|
||||
'def step(x: int, *, enabled=False):\n return x + 1\n',
|
||||
'def step(x: str, *, enabled=True):\n return x + 1\n',
|
||||
'@other\ndef step(x: int, *, enabled=True):\n return x + 1\n']
|
||||
for source in variants:
|
||||
with self.subTest(source=source):
|
||||
self.assertNotEqual(original, compatibility._ast_fingerprint(ast.parse(source).body[0]))
|
||||
|
||||
def test_empty_fields_are_retained_and_debug_locations_do_not_change_structure(self):
|
||||
reference = ast.parse('def step(x):\n return x\n').body[0]
|
||||
structure = dict(compatibility._ast_structure(reference)[1])
|
||||
self.assertEqual(structure['decorator_list'], [])
|
||||
self.assertEqual(structure['type_params'], [])
|
||||
self.assertIsNone(structure['returns'])
|
||||
changed = copy.deepcopy(reference)
|
||||
ast.increment_lineno(changed, 100)
|
||||
self.assertEqual(compatibility._ast_fingerprint(reference), compatibility._ast_fingerprint(changed))
|
||||
|
||||
def test_unknown_semantic_dependency_change_still_rejects(self):
|
||||
reference = ast.parse('def step(x):\n return x + 1\n').body[0]
|
||||
source = 'def step(x):\n return x + 2\n'
|
||||
with tempfile.TemporaryDirectory(prefix='krea-native-ast-') as folder:
|
||||
path = Path(folder) / 'dependency.py'
|
||||
path.write_text(source, encoding='utf-8')
|
||||
module = ModuleType('owned_ast_dependency_fixture')
|
||||
module.__file__ = str(path)
|
||||
exec(compile(source, str(path), 'exec', dont_inherit=True), module.__dict__)
|
||||
with self.assertRaisesRegex(RuntimeError, 'dependency source is unverified'):
|
||||
compatibility._torch_source(module, {'step': compatibility._ast_fingerprint(reference)})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user