diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 0e9bd98aa1fc31..ec378ee9e3e109 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -128,12 +128,22 @@ def __get_openssl_constructor(name): # Prefer our builtin blake2 implementation. return __get_builtin_constructor(name) try: - # MD5, SHA1, and SHA2 are in all supported OpenSSL versions - # SHA3/shake are available in OpenSSL 1.1.1+ + # Fetch the OpenSSL hash function if it exists, + # independently of the context security policy. f = getattr(_hashlib, 'openssl_' + name) - # Allow the C module to raise ValueError. The function will be - # defined but the hash not actually available. Don't fall back to - # builtin if the current security policy blocks a digest, bpo#40695. + # Check if the context security policy blocks the digest or not + # by allowing the C module to raise a ValueError. The function + # will be defined but the hash will not be available at runtime. + # + # We use "usedforsecurity=False" to prevent falling back to the + # built-in function in case the security policy does not allow it. + # + # Note that this only affects the explicit named constructors, + # and not the algorithms exposed through hashlib.new() which + # can still be resolved to a built-in function even if the + # current security policy does not allow it. + # + # See https://github.com/python/cpython/issues/84872. f(usedforsecurity=False) # Use the C function directly (very fast) return f diff --git a/Lib/test/support/hashlib_helper.py b/Lib/test/support/hashlib_helper.py index 7032257b06877a..96be74e4105c18 100644 --- a/Lib/test/support/hashlib_helper.py +++ b/Lib/test/support/hashlib_helper.py @@ -1,45 +1,187 @@ +import contextlib +import enum import functools -import hashlib import importlib +import inspect import unittest -from test.support.import_helper import import_module +import unittest.mock +from test.support import import_helper +from types import MappingProxyType -try: - import _hashlib -except ImportError: - _hashlib = None -try: - import _hmac -except ImportError: - _hmac = None +def try_import_module(module_name): + """Try to import a module and return None on failure.""" + try: + return importlib.import_module(module_name) + except ImportError: + return None -def requires_hashlib(): - return unittest.skipIf(_hashlib is None, "requires _hashlib") +class HID(enum.StrEnum): + """Enumeration containing the canonical digest names. + Those names should only be used by hashlib.new() or hmac.new(). + Their support by _hashlib.new() is not necessarily guaranteed. + """ -def requires_builtin_hmac(): - return unittest.skipIf(_hmac is None, "requires _hmac") + md5 = enum.auto() + sha1 = enum.auto() + sha224 = enum.auto() + sha256 = enum.auto() + sha384 = enum.auto() + sha512 = enum.auto() -def _missing_hash(digestname, implementation=None, *, exc=None): - parts = ["missing", implementation, f"hash algorithm: {digestname!r}"] - msg = " ".join(filter(None, parts)) - raise unittest.SkipTest(msg) from exc + sha3_224 = enum.auto() + sha3_256 = enum.auto() + sha3_384 = enum.auto() + sha3_512 = enum.auto() + shake_128 = enum.auto() + shake_256 = enum.auto() -def _openssl_availabillity(digestname, *, usedforsecurity): - try: - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - except AttributeError: - assert _hashlib is None - _missing_hash(digestname, "OpenSSL") - except ValueError as exc: - _missing_hash(digestname, "OpenSSL", exc=exc) + blake2s = enum.auto() + blake2b = enum.auto() + + def __repr__(self): + return str(self) + + @property + def is_xof(self): + """Indicate whether the hash is an extendable-output hash function.""" + return self.startswith("shake_") + + @property + def is_keyed(self): + """Indicate whether the hash is a keyed hash function.""" + return self.startswith("blake2") -def _decorate_func_or_class(func_or_class, decorator_func): +CANONICAL_DIGEST_NAMES = frozenset(map(str, HID.__members__)) +NON_HMAC_DIGEST_NAMES = frozenset(( + HID.shake_128, HID.shake_256, + HID.blake2s, HID.blake2b, +)) + + +class HashInfo: + """Dataclass storing explicit hash constructor names. + + - *builtin* is the fully-qualified name for the explicit HACL* + hash constructor function, e.g., "_md5.md5". + + - *openssl* is the name of the "_hashlib" module method for the explicit + OpenSSL hash constructor function, e.g., "openssl_md5". + + - *hashlib* is the name of the "hashlib" module method for the explicit + hash constructor function, e.g., "md5". + """ + + def __init__(self, builtin, openssl=None, hashlib=None): + assert isinstance(builtin, str), builtin + assert len(builtin.split(".")) == 2, builtin + + self.builtin = builtin + self.builtin_module_name, self.builtin_method_name = ( + self.builtin.split(".", maxsplit=1) + ) + + assert openssl is None or openssl.startswith("openssl_") + self.openssl = self.openssl_method_name = openssl + self.openssl_module_name = "_hashlib" if openssl else None + + assert hashlib is None or isinstance(hashlib, str) + self.hashlib = self.hashlib_method_name = hashlib + self.hashlib_module_name = "hashlib" if hashlib else None + + def module_name(self, implementation): + match implementation: + case "builtin": + return self.builtin_module_name + case "openssl": + return self.openssl_module_name + case "hashlib": + return self.hashlib_module_name + raise AssertionError(f"invalid implementation {implementation}") + + def method_name(self, implementation): + match implementation: + case "builtin": + return self.builtin_method_name + case "openssl": + return self.openssl_method_name + case "hashlib": + return self.hashlib_method_name + raise AssertionError(f"invalid implementation {implementation}") + + def fullname(self, implementation): + """Get the fully qualified name of a given implementation. + + This returns a string of the form "MODULE_NAME.METHOD_NAME" or None + if the hash function does not have a corresponding implementation. + + *implementation* must be "builtin", "openssl" or "hashlib". + """ + module_name = self.module_name(implementation) + method_name = self.method_name(implementation) + if module_name is None or method_name is None: + return None + return f"{module_name}.{method_name}" + + +# Mapping from a "canonical" name to a pair (HACL*, _hashlib.*, hashlib.*) +# constructors. If the constructor name is None, then this means that the +# algorithm can only be used by the "agile" new() interfaces. +_EXPLICIT_CONSTRUCTORS = MappingProxyType({ # fmt: skip + HID.md5: HashInfo("_md5.md5", "openssl_md5", "md5"), + HID.sha1: HashInfo("_sha1.sha1", "openssl_sha1", "sha1"), + HID.sha224: HashInfo("_sha2.sha224", "openssl_sha224", "sha224"), + HID.sha256: HashInfo("_sha2.sha256", "openssl_sha256", "sha256"), + HID.sha384: HashInfo("_sha2.sha384", "openssl_sha384", "sha384"), + HID.sha512: HashInfo("_sha2.sha512", "openssl_sha512", "sha512"), + HID.sha3_224: HashInfo( + "_sha3.sha3_224", "openssl_sha3_224", "sha3_224" + ), + HID.sha3_256: HashInfo( + "_sha3.sha3_256", "openssl_sha3_256", "sha3_256" + ), + HID.sha3_384: HashInfo( + "_sha3.sha3_384", "openssl_sha3_384", "sha3_384" + ), + HID.sha3_512: HashInfo( + "_sha3.sha3_512", "openssl_sha3_512", "sha3_512" + ), + HID.shake_128: HashInfo( + "_sha3.shake_128", "openssl_shake_128", "shake_128" + ), + HID.shake_256: HashInfo( + "_sha3.shake_256", "openssl_shake_256", "shake_256" + ), + HID.blake2s: HashInfo("_blake2.blake2s", None, "blake2s"), + HID.blake2b: HashInfo("_blake2.blake2b", None, "blake2b"), +}) +assert _EXPLICIT_CONSTRUCTORS.keys() == CANONICAL_DIGEST_NAMES +get_hash_info = _EXPLICIT_CONSTRUCTORS.__getitem__ + +# Mapping from canonical hash names to their explicit HACL* HMAC constructor. +# There is currently no OpenSSL one-shot named function and there will likely +# be none in the future. +_EXPLICIT_HMAC_CONSTRUCTORS = { + HID(name): f"_hmac.compute_{name}" + for name in CANONICAL_DIGEST_NAMES +} +# Neither HACL* nor OpenSSL supports HMAC over XOFs. +_EXPLICIT_HMAC_CONSTRUCTORS[HID.shake_128] = None +_EXPLICIT_HMAC_CONSTRUCTORS[HID.shake_256] = None +# Strictly speaking, HMAC-BLAKE is meaningless as BLAKE2 is already a +# keyed hash function. However, as it's exposed by HACL*, we test it. +_EXPLICIT_HMAC_CONSTRUCTORS[HID.blake2s] = '_hmac.compute_blake2s_32' +_EXPLICIT_HMAC_CONSTRUCTORS[HID.blake2b] = '_hmac.compute_blake2b_32' +_EXPLICIT_HMAC_CONSTRUCTORS = MappingProxyType(_EXPLICIT_HMAC_CONSTRUCTORS) +assert _EXPLICIT_HMAC_CONSTRUCTORS.keys() == CANONICAL_DIGEST_NAMES + + +def _decorate_func_or_class(decorator_func, func_or_class): if not isinstance(func_or_class, type): return decorator_func(func_or_class) @@ -57,112 +199,216 @@ def setUpClass(cls): return decorated_class -def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): - """Decorator raising SkipTest if a hashing algorithm is not available. +def _chain_decorators(decorators): + """Obtain a decorator by chaining multiple decorators. - The hashing algorithm may be missing, blocked by a strict crypto policy, - or Python may be configured with `--with-builtin-hashlib-hashes=no`. + The decorators are applied in the order they are given. + """ + def decorator_func(func): + return functools.reduce(lambda w, deco: deco(w), decorators, func) + return functools.partial(_decorate_func_or_class, decorator_func) - If 'openssl' is True, then the decorator checks that OpenSSL provides - the algorithm. Otherwise the check falls back to (optional) built-in - HACL* implementations. - The usedforsecurity flag is passed to the constructor but has no effect - on HACL* implementations. +def _ensure_wrapper_signature(wrapper, wrapped): + """Ensure that a wrapper has the same signature as the wrapped function. - Examples of exceptions being suppressed: - ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS - ValueError: unsupported hash type md4 + This is used to guarantee that a TypeError raised due to a bad API call + is raised consistently (using variadic signatures would hide such errors). """ - if openssl and _hashlib is not None: - def test_availability(): - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - else: - def test_availability(): - hashlib.new(digestname, usedforsecurity=usedforsecurity) + try: + wrapped_sig = inspect.signature(wrapped) + except ValueError: # built-in signature cannot be found + return + + wrapper_sig = inspect.signature(wrapper) + if wrapped_sig != wrapper_sig: + fullname = f"{wrapped.__module__}.{wrapped.__qualname__}" + raise AssertionError( + f"signature for {fullname}() is incorrect:\n" + f" expect: {wrapped_sig}\n" + f" actual: {wrapper_sig}" + ) - def decorator_func(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - test_availability() - except ValueError as exc: - _missing_hash(digestname, exc=exc) - return func(*args, **kwargs) - return wrapper - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator +def requires_hashlib(): + _hashlib = try_import_module("_hashlib") + return unittest.skipIf(_hashlib is None, "requires _hashlib") -def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): - """Decorator raising SkipTest if an OpenSSL hashing algorithm is missing. +def requires_builtin_hmac(): + _hmac = try_import_module("_hmac") + return unittest.skipIf(_hmac is None, "requires _hmac") - The hashing algorithm may be missing or blocked by a strict crypto policy. - """ - def decorator_func(func): - @requires_hashlib() # avoid checking at each call - @functools.wraps(func) - def wrapper(*args, **kwargs): - _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) - return func(*args, **kwargs) - return wrapper - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator +class SkipNoHash(unittest.SkipTest): + """A SkipTest exception raised when a hash is not available.""" + + def __init__(self, digestname, implementation=None, interface=None): + parts = ["missing", implementation, f"hash algorithm {digestname!r}"] + if interface is not None: + parts.append(f"for {interface}") + super().__init__(" ".join(filter(None, parts))) -def find_openssl_hashdigest_constructor(digestname, *, usedforsecurity=True): - """Find the OpenSSL hash function constructor by its name.""" +def _hashlib_new(digestname, openssl, /, **kwargs): + """Check availability of [hashlib|_hashlib].new(digestname, **kwargs). + + If *openssl* is True, module is "_hashlib" (C extension module), + otherwise it is "hashlib" (pure Python interface). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ assert isinstance(digestname, str), digestname - _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) - # This returns a function of the form _hashlib.openssl_ and - # not a lambda function as it is rejected by _hashlib.hmac_new(). - return getattr(_hashlib, f"openssl_{digestname}") + # Re-import 'hashlib' in case it was mocked, but propagate + # exceptions as it should be unconditionally available. + hashlib = importlib.import_module("hashlib") + # re-import '_hashlib' in case it was mocked + _hashlib = try_import_module("_hashlib") + module = _hashlib if openssl and _hashlib is not None else hashlib + try: + module.new(digestname, **kwargs) + except ValueError as exc: + interface = f"{module.__name__}.new" + raise SkipNoHash(digestname, interface=interface) from exc + return functools.partial(module.new, digestname) -def requires_builtin_hashdigest( - module_name, digestname, *, usedforsecurity=True -): - """Decorator raising SkipTest if a HACL* hashing algorithm is missing. +def _builtin_hash(module_name, digestname, /, **kwargs): + """Check availability of .(**kwargs). - The *module_name* is the C extension module name based on HACL*. - The *digestname* is one of its member, e.g., 'md5'. + + The constructor function is returned, or SkipTest is raised if none exists. + """ + assert isinstance(module_name, str), module_name + assert isinstance(digestname, str), digestname + fullname = f'{module_name}.{digestname}' + try: + builtin_module = importlib.import_module(module_name) + except ImportError as exc: + raise SkipNoHash(fullname, "builtin") from exc + try: + constructor = getattr(builtin_module, digestname) + except AttributeError as exc: + raise SkipNoHash(fullname, "builtin") from exc + try: + constructor(**kwargs) + except ValueError as exc: + raise SkipNoHash(fullname, "builtin") from exc + return constructor + + +def _openssl_new(digestname, /, **kwargs): + """Check availability of _hashlib.new(digestname, **kwargs). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. """ + assert isinstance(digestname, str), digestname + try: + # re-import '_hashlib' in case it was mocked + _hashlib = importlib.import_module("_hashlib") + except ImportError as exc: + raise SkipNoHash(digestname, "openssl") from exc + try: + _hashlib.new(digestname, **kwargs) + except ValueError as exc: + raise SkipNoHash(digestname, interface="_hashlib.new") from exc + return functools.partial(_hashlib.new, digestname) + + +def _openssl_hash(digestname, /, **kwargs): + """Check availability of _hashlib.openssl_(**kwargs). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ + assert isinstance(digestname, str), digestname + fullname = f"_hashlib.openssl_{digestname}" + try: + # re-import '_hashlib' in case it was mocked + _hashlib = importlib.import_module("_hashlib") + except ImportError as exc: + raise SkipNoHash(fullname, "openssl") from exc + try: + constructor = getattr(_hashlib, f"openssl_{digestname}", None) + except AttributeError as exc: + raise SkipNoHash(fullname, "openssl") from exc + try: + constructor(**kwargs) + except ValueError as exc: + raise SkipNoHash(fullname, "openssl") from exc + return constructor + + +def _make_requires_hashdigest_decorator(test, /, *test_args, **test_kwargs): def decorator_func(func): @functools.wraps(func) def wrapper(*args, **kwargs): - module = import_module(module_name) - try: - getattr(module, digestname) - except AttributeError: - fullname = f'{module_name}.{digestname}' - _missing_hash(fullname, implementation="HACL") + test(*test_args, **test_kwargs) return func(*args, **kwargs) return wrapper + return functools.partial(_decorate_func_or_class, decorator_func) + + +def requires_hashdigest(digestname, openssl=None, *, usedforsecurity=True): + """Decorator raising SkipTest if a hashing algorithm is not available. + + The hashing algorithm may be missing, blocked by a strict crypto policy, + or Python may be configured with `--with-builtin-hashlib-hashes=no`. + + If 'openssl' is True, then the decorator checks that OpenSSL provides + the algorithm. Otherwise the check falls back to (optional) built-in + HACL* implementations. + + The usedforsecurity flag is passed to the constructor but has no effect + on HACL* implementations. + + Examples of exceptions being suppressed: + ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS + ValueError: unsupported hash type md4 + """ + return _make_requires_hashdigest_decorator( + _hashlib_new, digestname, openssl, usedforsecurity=usedforsecurity + ) + + +def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): + """Decorator raising SkipTest if an OpenSSL hashing algorithm is missing. - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator + The hashing algorithm may be missing or blocked by a strict crypto policy. + """ + return _make_requires_hashdigest_decorator( + _openssl_new, digestname, usedforsecurity=usedforsecurity + ) -def find_builtin_hashdigest_constructor( +def requires_builtin_hashdigest( module_name, digestname, *, usedforsecurity=True ): - """Find the HACL* hash function constructor. + """Decorator raising SkipTest if a HACL* hashing algorithm is missing. - The *module_name* is the C extension module name based on HACL*. - The *digestname* is one of its member, e.g., 'md5'. """ - module = import_module(module_name) - try: - constructor = getattr(module, digestname) - constructor(b'', usedforsecurity=usedforsecurity) - except (AttributeError, TypeError, ValueError): - _missing_hash(f'{module_name}.{digestname}', implementation="HACL") - return constructor + return _make_requires_hashdigest_decorator( + _builtin_hash, module_name, digestname, usedforsecurity=usedforsecurity + ) + + +def requires_builtin_hashes(*ignored, usedforsecurity=True): + """Decorator raising SkipTest if one HACL* hashing algorithm is missing.""" + return _chain_decorators(( + requires_builtin_hashdigest( + api.builtin_module_name, + api.builtin_method_name, + usedforsecurity=usedforsecurity, + ) + for name, api in _EXPLICIT_CONSTRUCTORS.items() + if name not in ignored + )) class HashFunctionsTrait: @@ -178,19 +424,29 @@ class HashFunctionsTrait: implementation of HMAC). """ - ALGORITHMS = [ + DIGEST_NAMES = [ 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', ] - # Default 'usedforsecurity' to use when looking up a hash function. + # Default 'usedforsecurity' to use when checking a hash function. + # When the trait properties are callables (e.g., _md5.md5) and + # not strings, they must be called with the same 'usedforsecurity'. usedforsecurity = True - def _find_constructor(self, name): + @classmethod + def setUpClass(cls): + super().setUpClass() + assert CANONICAL_DIGEST_NAMES.issuperset(cls.DIGEST_NAMES) + + def is_valid_digest_name(self, digestname): + self.assertIn(digestname, self.DIGEST_NAMES) + + def _find_constructor(self, digestname): # By default, a missing algorithm skips the test that uses it. - self.assertIn(name, self.ALGORITHMS) - self.skipTest(f"missing hash function: {name}") + self.is_valid_digest_name(digestname) + self.skipTest(f"missing hash function: {digestname}") @property def md5(self): @@ -239,9 +495,9 @@ class NamedHashFunctionsTrait(HashFunctionsTrait): Hash functions are available if and only if they are available in hashlib. """ - def _find_constructor(self, name): - self.assertIn(name, self.ALGORITHMS) - return name + def _find_constructor(self, digestname): + self.is_valid_digest_name(digestname) + return digestname class OpenSSLHashFunctionsTrait(HashFunctionsTrait): @@ -250,11 +506,11 @@ class OpenSSLHashFunctionsTrait(HashFunctionsTrait): Hash functions are available if and only if they are available in _hashlib. """ - def _find_constructor(self, name): - self.assertIn(name, self.ALGORITHMS) - return find_openssl_hashdigest_constructor( - name, usedforsecurity=self.usedforsecurity - ) + def _find_constructor(self, digestname): + self.is_valid_digest_name(digestname) + # This returns a function of the form _hashlib.openssl_ and + # not a lambda function as it is rejected by _hashlib.hmac_new(). + return _openssl_hash(digestname, usedforsecurity=self.usedforsecurity) class BuiltinHashFunctionsTrait(HashFunctionsTrait): @@ -265,65 +521,276 @@ class BuiltinHashFunctionsTrait(HashFunctionsTrait): is not since the former is unconditionally built. """ - def _find_constructor_in(self, module, name): - self.assertIn(name, self.ALGORITHMS) - return find_builtin_hashdigest_constructor(module, name) + def _find_constructor(self, digestname): + self.is_valid_digest_name(digestname) + info = _EXPLICIT_CONSTRUCTORS[digestname] + return _builtin_hash( + info.builtin_module_name, + info.builtin_method_name, + usedforsecurity=self.usedforsecurity, + ) - @property - def md5(self): - return self._find_constructor_in("_md5", "md5") - @property - def sha1(self): - return self._find_constructor_in("_sha1", "sha1") +def find_gil_minsize(modules_names, default=2048): + """Get the largest GIL_MINSIZE value for the given cryptographic modules. - @property - def sha224(self): - return self._find_constructor_in("_sha2", "sha224") + The valid module names are the following: - @property - def sha256(self): - return self._find_constructor_in("_sha2", "sha256") + - _hashlib + - _md5, _sha1, _sha2, _sha3, _blake2 + - _hmac + """ + sizes = [] + for module_name in modules_names: + module = try_import_module(module_name) + if module is not None: + sizes.append(getattr(module, '_GIL_MINSIZE', default)) + return max(sizes, default=default) - @property - def sha384(self): - return self._find_constructor_in("_sha2", "sha384") - @property - def sha512(self): - return self._find_constructor_in("_sha2", "sha512") +def _block_openssl_hash_new(blocked_name): + """Block OpenSSL implementation of _hashlib.new().""" + assert isinstance(blocked_name, str), blocked_name - @property - def sha3_224(self): - return self._find_constructor_in("_sha3", "sha3_224") + # re-import '_hashlib' in case it was mocked + if (_hashlib := try_import_module("_hashlib")) is None: + return contextlib.nullcontext() - @property - def sha3_256(self): - return self._find_constructor_in("_sha3","sha3_256") + @functools.wraps(wrapped := _hashlib.new) + def _hashlib_new(name, data=b'', *, usedforsecurity=True, string=None): + if name == blocked_name: + raise _hashlib.UnsupportedDigestmodError(blocked_name) + return wrapped(name, data, + usedforsecurity=usedforsecurity, string=string) - @property - def sha3_384(self): - return self._find_constructor_in("_sha3","sha3_384") + _ensure_wrapper_signature(_hashlib_new, wrapped) + return unittest.mock.patch('_hashlib.new', _hashlib_new) - @property - def sha3_512(self): - return self._find_constructor_in("_sha3","sha3_512") +def _block_openssl_hmac_new(blocked_name): + """Block OpenSSL HMAC-HASH implementation.""" + assert isinstance(blocked_name, str), blocked_name -def find_gil_minsize(modules_names, default=2048): - """Get the largest GIL_MINSIZE value for the given cryptographic modules. + # re-import '_hashlib' in case it was mocked + if (_hashlib := try_import_module("_hashlib")) is None: + return contextlib.nullcontext() - The valid module names are the following: + @functools.wraps(wrapped := _hashlib.hmac_new) + def wrapper(key, msg=b'', digestmod=None): + if digestmod == blocked_name: + raise _hashlib.UnsupportedDigestmodError(blocked_name) + return wrapped(key, msg, digestmod) - - _hashlib - - _md5, _sha1, _sha2, _sha3, _blake2 - - _hmac + _ensure_wrapper_signature(wrapper, wrapped) + return unittest.mock.patch('_hashlib.hmac_new', wrapper) + + +def _block_openssl_hmac_digest(blocked_name): + """Block OpenSSL HMAC-HASH one-shot digest implementation.""" + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hashlib' in case it was mocked + if (_hashlib := try_import_module("_hashlib")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hashlib.hmac_digest) + def _hashlib_hmac_digest(key, msg, digest): + if digest == blocked_name: + raise _hashlib.UnsupportedDigestmodError(blocked_name) + return wrapped(key, msg, digest) + + _ensure_wrapper_signature(_hashlib_hmac_digest, wrapped) + return unittest.mock.patch('_hashlib.hmac_digest', _hashlib_hmac_digest) + + +def _block_builtin_hash_new(name): + """Block a buitin-in hash name from the hashlib.new() interface.""" + assert isinstance(name, str), name + assert name.lower() == name, f"invalid name: {name}" + assert name in HID, f"invalid hash: {name}" + + # Re-import 'hashlib' in case it was mocked + hashlib = importlib.import_module('hashlib') + builtin_constructor_cache = getattr(hashlib, '__builtin_constructor_cache') + builtin_constructor_cache_mock = builtin_constructor_cache.copy() + builtin_constructor_cache_mock.pop(name, None) + builtin_constructor_cache_mock.pop(name.upper(), None) + + # __get_builtin_constructor() imports the HACL* modules on demand, + # so we need to block the possibility of importing it, but only + # during the call to __get_builtin_constructor(). + get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') + builtin_module_name = _EXPLICIT_CONSTRUCTORS[name].builtin_module_name + + @functools.wraps(get_builtin_constructor) + def get_builtin_constructor_mock(name): + with import_helper.isolated_modules(): + sys = importlib.import_module("sys") + sys.modules[builtin_module_name] = None # block module's import + return get_builtin_constructor(name) + + return unittest.mock.patch.multiple( + hashlib, + __get_builtin_constructor=get_builtin_constructor_mock, + __builtin_constructor_cache=builtin_constructor_cache_mock + ) + + +def _block_builtin_hmac_new(blocked_name): + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hmac' in case it was mocked + if (_hmac := try_import_module("_hmac")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hmac.new) + def _hmac_new(key, msg=None, digestmod=None): + if digestmod == blocked_name: + raise _hmac.UnknownHashError(blocked_name) + return wrapped(key, msg, digestmod) + + _ensure_wrapper_signature(_hmac_new, wrapped) + return unittest.mock.patch('_hmac.new', _hmac_new) + + +def _block_builtin_hmac_digest(blocked_name): + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hmac' in case it was mocked + if (_hmac := try_import_module("_hmac")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hmac.compute_digest) + def _hmac_compute_digest(key, msg, digest): + if digest == blocked_name: + raise _hmac.UnknownHashError(blocked_name) + return wrapped(key, msg, digest) + + _ensure_wrapper_signature(_hmac_compute_digest, wrapped) + return unittest.mock.patch('_hmac.compute_digest', _hmac_compute_digest) + + +def _make_hash_constructor_blocker(name, dummy, implementation): + info = _EXPLICIT_CONSTRUCTORS[name] + module_name = info.module_name(implementation) + method_name = info.method_name(implementation) + if module_name is None or method_name is None: + # function shouldn't exist for this implementation + return contextlib.nullcontext() + + try: + module = importlib.import_module(module_name) + except ImportError: + # module is already disabled + return contextlib.nullcontext() + + wrapped = getattr(module, method_name) + wrapper = functools.wraps(wrapped)(dummy) + _ensure_wrapper_signature(wrapper, wrapped) + return unittest.mock.patch(info.fullname(implementation), wrapper) + + +def _block_hashlib_hash_constructor(name): + """Block explicit public constructors.""" + def dummy(data=b'', *, usedforsecurity=True, string=None): + raise ValueError(f"blocked explicit public hash name: {name}") + + return _make_hash_constructor_blocker(name, dummy, 'hashlib') + + +def _block_openssl_hash_constructor(name): + """Block explicit OpenSSL constructors.""" + def dummy(data=b'', *, usedforsecurity=True, string=None): + raise ValueError(f"blocked explicit OpenSSL hash name: {name}") + return _make_hash_constructor_blocker(name, dummy, 'openssl') + + +def _block_builtin_hash_constructor(name): + """Block explicit HACL* constructors.""" + def dummy(data=b'', *, usedforsecurity=True, string=b''): + raise ValueError(f"blocked explicit builtin hash name: {name}") + return _make_hash_constructor_blocker(name, dummy, 'builtin') + + +def _block_builtin_hmac_constructor(name): + """Block explicit HACL* HMAC constructors.""" + fullname = _EXPLICIT_HMAC_CONSTRUCTORS[name] + if fullname is None: + # function shouldn't exist for this implementation + return contextlib.nullcontext() + assert fullname.count('.') == 1, fullname + module_name, method = fullname.split('.', maxsplit=1) + assert module_name == '_hmac', module_name + try: + module = importlib.import_module(module_name) + except ImportError: + # module is already disabled + return contextlib.nullcontext() + @functools.wraps(wrapped := getattr(module, method)) + def wrapper(key, obj): + raise ValueError(f"blocked hash name: {name}") + _ensure_wrapper_signature(wrapper, wrapped) + return unittest.mock.patch(fullname, wrapper) + + +@contextlib.contextmanager +def block_algorithm(name, *, allow_openssl=False, allow_builtin=False): + """Block a hash algorithm for both hashing and HMAC. + + Be careful with this helper as a function may be allowed, but can + still raise a ValueError at runtime if the OpenSSL security policy + disables it, e.g., if allow_openssl=True and FIPS mode is on. """ - sizes = [] - for module_name in modules_names: - try: - module = importlib.import_module(module_name) - except ImportError: - continue - sizes.append(getattr(module, '_GIL_MINSIZE', default)) - return max(sizes, default=default) + with contextlib.ExitStack() as stack: + if not (allow_openssl or allow_builtin): + # Named constructors have a different behavior in the sense + # that they are either built-ins or OpenSSL ones, but not + # "agile" ones (namely once "hashlib" has been imported, + # they are fixed). + # + # If OpenSSL is not available, hashes fall back to built-in ones, + # in which case we don't need to block the explicit public hashes + # as they will call a mocked one. + # + # If OpenSSL is available, hashes fall back to "openssl_*" ones, + # except for BLAKE2b and BLAKE2s. + stack.enter_context(_block_hashlib_hash_constructor(name)) + elif ( + # In FIPS mode, hashlib.() functions may raise if they use + # the OpenSSL implementation, except with usedforsecurity=False. + # However, blocking such functions also means blocking them + # so we again need to block them if we want to. + (_hashlib := try_import_module("_hashlib")) + and _hashlib.get_fips_mode() + and not allow_openssl + ) or ( + # Without OpenSSL, hashlib.() functions are aliases + # to built-in functions, so both of them must be blocked + # as the module may have been imported before the HACL ones. + not (_hashlib := try_import_module("_hashlib")) + and not allow_builtin + ): + stack.enter_context(_block_hashlib_hash_constructor(name)) + + if not allow_openssl: + # _hashlib.new() + stack.enter_context(_block_openssl_hash_new(name)) + # _hashlib.openssl_*() + stack.enter_context(_block_openssl_hash_constructor(name)) + # _hashlib.hmac_new() + stack.enter_context(_block_openssl_hmac_new(name)) + # _hashlib.hmac_digest() + stack.enter_context(_block_openssl_hmac_digest(name)) + + if not allow_builtin: + # __get_builtin_constructor(name) + stack.enter_context(_block_builtin_hash_new(name)) + # .() + stack.enter_context(_block_builtin_hash_constructor(name)) + # _hmac.new(..., name) + stack.enter_context(_block_builtin_hmac_new(name)) + # _hmac.compute_() + stack.enter_context(_block_builtin_hmac_constructor(name)) + # _hmac.compute_digest(..., name) + stack.enter_context(_block_builtin_hmac_digest(name)) + yield diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 0fecc95fec40fc..ffc207a11a9b55 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -477,13 +477,17 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): def check_file_digest(self, name, data, hexdigest): hexdigest = hexdigest.lower() - try: - hashlib.new(name) - except ValueError: - # skip, algorithm is blocked by security policy. - return - digests = [name] - digests.extend(self.constructors_to_test[name]) + digests = [] + for digest in [name, *self.constructors_to_test[name]]: + try: + if callable(digest): + digest(b"") + else: + hashlib.new(digest) + except ValueError: + # skip, algorithm is blocked by security policy. + continue + digests.append(digest) with tempfile.TemporaryFile() as f: f.write(data) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e48a2464ee5977..92b3ef26cd979a 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1,6 +1,8 @@ import contextlib import errno import importlib +import itertools +import inspect import io import logging import os @@ -17,6 +19,7 @@ import warnings from test import support +from test.support import hashlib_helper from test.support import import_helper from test.support import os_helper from test.support import script_helper @@ -818,5 +821,238 @@ def test_linked_to_musl(self): # SuppressCrashReport +@hashlib_helper.requires_builtin_hashes() +class TestHashlibSupport(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.hashlib = import_helper.import_module("hashlib") + cls.hmac = import_helper.import_module("hmac") + + # All C extension modules must be present since blocking + # the built-in implementation while allowing OpenSSL or vice-versa + # may result in failures depending on the exposed built-in hashes. + cls._hashlib = import_helper.import_module("_hashlib") + cls._hmac = import_helper.import_module("_hmac") + cls._md5 = import_helper.import_module("_md5") + + def skip_if_fips_mode(self): + if self._hashlib.get_fips_mode(): + self.skipTest("disabled in FIPS mode") + + def skip_if_not_fips_mode(self): + if not self._hashlib.get_fips_mode(): + self.skipTest("requires FIPS mode") + + def check_context(self, disabled=True): + if disabled: + return self.assertRaises(ValueError) + return contextlib.nullcontext() + + def try_import_attribute(self, fullname, default=None): + if fullname is None: + return default + assert fullname.count('.') == 1, fullname + module_name, attribute = fullname.split('.', maxsplit=1) + try: + module = importlib.import_module(module_name) + except ImportError: + return default + try: + return getattr(module, attribute, default) + except TypeError: + return default + + def fetch_hash_function(self, name, implementation): + info = hashlib_helper.get_hash_info(name) + match implementation: + case "hashlib": + assert info.hashlib is not None, info + return getattr(self.hashlib, info.hashlib) + case "openssl": + try: + return getattr(self._hashlib, info.openssl, None) + except TypeError: + return None + fullname = info.fullname(implementation) + return self.try_import_attribute(fullname) + + def fetch_hmac_function(self, name): + fullname = hashlib_helper._EXPLICIT_HMAC_CONSTRUCTORS[name] + return self.try_import_attribute(fullname) + + def check_openssl_hash(self, name, *, disabled=True): + """Check that OpenSSL HASH interface is enabled/disabled.""" + with self.check_context(disabled): + _ = self._hashlib.new(name) + if do_hash := self.fetch_hash_function(name, "openssl"): + self.assertStartsWith(do_hash.__name__, 'openssl_') + with self.check_context(disabled): + _ = do_hash(b"") + + def check_openssl_hmac(self, name, *, disabled=True): + """Check that OpenSSL HMAC interface is enabled/disabled.""" + if name in hashlib_helper.NON_HMAC_DIGEST_NAMES: + # HMAC-BLAKE and HMAC-SHAKE raise a ValueError as they are not + # supported at all (they do not make any sense in practice). + with self.assertRaises(ValueError): + self._hashlib.hmac_digest(b"", b"", name) + else: + with self.check_context(disabled): + _ = self._hashlib.hmac_digest(b"", b"", name) + # OpenSSL does not provide one-shot explicit HMAC functions + + def check_builtin_hash(self, name, *, disabled=True): + """Check that HACL* HASH interface is enabled/disabled.""" + if do_hash := self.fetch_hash_function(name, "builtin"): + self.assertEqual(do_hash.__name__, name) + with self.check_context(disabled): + _ = do_hash(b"") + + def check_builtin_hmac(self, name, *, disabled=True): + """Check that HACL* HMAC interface is enabled/disabled.""" + if name in hashlib_helper.NON_HMAC_DIGEST_NAMES: + # HMAC-BLAKE and HMAC-SHAKE raise a ValueError as they are not + # supported at all (they do not make any sense in practice). + with self.assertRaises(ValueError): + self._hmac.compute_digest(b"", b"", name) + else: + with self.check_context(disabled): + _ = self._hmac.compute_digest(b"", b"", name) + + with self.check_context(disabled): + _ = self._hmac.new(b"", b"", name) + + if do_hmac := self.fetch_hmac_function(name): + self.assertStartsWith(do_hmac.__name__, 'compute_') + with self.check_context(disabled): + _ = do_hmac(b"", b"") + else: + self.assertIn(name, hashlib_helper.NON_HMAC_DIGEST_NAMES) + + @support.subTests( + ('name', 'allow_openssl', 'allow_builtin'), + itertools.product( + hashlib_helper.CANONICAL_DIGEST_NAMES, + [True, False], + [True, False], + ) + ) + def test_disable_hash(self, name, allow_openssl, allow_builtin): + # In FIPS mode, the function may be available but would still need + # to raise a ValueError, so we will test the helper separately. + self.skip_if_fips_mode() + flags = dict(allow_openssl=allow_openssl, allow_builtin=allow_builtin) + is_fully_disabled = not allow_builtin and not allow_openssl + + with hashlib_helper.block_algorithm(name, **flags): + # OpenSSL's blake2s and blake2b are unknown names + # when only the OpenSSL interface is available. + if allow_openssl and not allow_builtin: + aliases = {'blake2s': 'blake2s256', 'blake2b': 'blake2b512'} + name_for_hashlib_new = aliases.get(name, name) + else: + name_for_hashlib_new = name + + with self.check_context(is_fully_disabled): + _ = self.hashlib.new(name_for_hashlib_new) + + # Since _hashlib is present, explicit blake2b/blake2s constructors + # use the built-in implementation, while others (since we are not + # in FIPS mode and since _hashlib exists) use the OpenSSL function. + with self.check_context(is_fully_disabled): + _ = getattr(self.hashlib, name)() + + self.check_openssl_hash(name, disabled=not allow_openssl) + self.check_builtin_hash(name, disabled=not allow_builtin) + + if name not in hashlib_helper.NON_HMAC_DIGEST_NAMES: + with self.check_context(is_fully_disabled): + _ = self.hmac.new(b"", b"", name) + with self.check_context(is_fully_disabled): + _ = self.hmac.HMAC(b"", b"", name) + with self.check_context(is_fully_disabled): + _ = self.hmac.digest(b"", b"", name) + + self.check_openssl_hmac(name, disabled=not allow_openssl) + self.check_builtin_hmac(name, disabled=not allow_builtin) + + @hashlib_helper.block_algorithm("md5") + def test_disable_hash_md5_in_fips_mode(self): + self.skip_if_not_fips_mode() + + self.assertRaises(ValueError, self.hashlib.new, "md5") + self.assertRaises(ValueError, self._hashlib.new, "md5") + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self._hashlib.openssl_md5) + + kwargs = dict(usedforsecurity=True) + self.assertRaises(ValueError, self.hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self._hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self.hashlib.md5, **kwargs) + self.assertRaises(ValueError, self._hashlib.openssl_md5, **kwargs) + + @hashlib_helper.block_algorithm("md5", allow_openssl=True) + def test_disable_hash_md5_in_fips_mode_allow_openssl(self): + self.skip_if_not_fips_mode() + # Allow the OpenSSL interface to be used but not the HACL* one. + # hashlib.new("md5") is dispatched to hashlib.openssl_md5() + self.assertRaises(ValueError, self.hashlib.new, "md5") + # dispatched to hashlib.openssl_md5() in FIPS mode + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, self._hashlib.HASH) + + # block_algorithm() does not mock hashlib.md5 and _hashlib.openssl_md5 + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + self.assertIs(hashlib_md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + # allow MD5 to be used in FIPS mode if usedforsecurity=False + h3 = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h3, self._hashlib.HASH) + + @hashlib_helper.block_algorithm("md5", allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_builtin(self): + self.skip_if_not_fips_mode() + # Allow the HACL* interface to be used but not the OpenSSL one. + h1 = self.hashlib.new("md5") # dispatched to _md5.md5() + self.assertNotIsInstance(h1, self._hashlib.HASH) + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, type(h1)) + + # block_algorithm() mocks hashlib.md5 and _hashlib.openssl_md5 + self.assertHasAttr(self.hashlib.md5, "__wrapped__") + self.assertHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + openssl_md5 = inspect.unwrap(self._hashlib.openssl_md5) + self.assertIs(hashlib_md5, openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self.hashlib.md5, + usedforsecurity=False) + + @hashlib_helper.block_algorithm("md5", + allow_openssl=True, + allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_all(self): + self.skip_if_not_fips_mode() + # hashlib.new() isn't blocked as it falls back to _md5.md5 + self.assertIsInstance(self.hashlib.new("md5"), self._md5.MD5Type) + self.assertRaises(ValueError, self._hashlib.new, "md5") + h = self._hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + self.assertIs(self.hashlib.md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + h = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + + if __name__ == '__main__': unittest.main()