From c3bfa0d6f3ac3cea78cc497a3c44002ea46437a1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 14 Jun 2025 00:21:35 +0100 Subject: [PATCH 01/15] Handle corner case: protocol vs classvar vs descriptor (#19277) Ref https://github.com/python/mypy/issues/19274 This is a bit ugly. But I propose to have this "hot-fix" until we have a proper overhaul of instance vs class variables. To be clear: attribute access already works correctly (on both `P` and `Type[P]`), but subtyping returns false because of ```python elif (IS_CLASSVAR in subflags) != (IS_CLASSVAR in superflags): return False ``` --- docs/source/protocols.rst | 47 +++++++++++++++++++++++++++++ mypy/subtypes.py | 12 +++++++- test-data/unit/check-protocols.test | 44 +++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/docs/source/protocols.rst b/docs/source/protocols.rst index ed8d94f62ef1..258cd4b0de56 100644 --- a/docs/source/protocols.rst +++ b/docs/source/protocols.rst @@ -352,6 +352,53 @@ the parameters are positional-only. Example (using the legacy syntax for generic copy_a = copy_b # OK copy_b = copy_a # Also OK +Binding of types in protocol attributes +*************************************** + +All protocol attributes annotations are treated as externally visible types +of those attributes. This means that for example callables are not bound, +and descriptors are not invoked: + +.. code-block:: python + + from typing import Callable, Protocol, overload + + class Integer: + @overload + def __get__(self, instance: None, owner: object) -> Integer: ... + @overload + def __get__(self, instance: object, owner: object) -> int: ... + # + + class Example(Protocol): + foo: Callable[[object], int] + bar: Integer + + ex: Example + reveal_type(ex.foo) # Revealed type is Callable[[object], int] + reveal_type(ex.bar) # Revealed type is Integer + +In other words, protocol attribute types are handled as they would appear in a +``self`` attribute annotation in a regular class. If you want some protocol +attributes to be handled as though they were defined at class level, you should +declare them explicitly using ``ClassVar[...]``. Continuing previous example: + +.. code-block:: python + + from typing import ClassVar + + class OtherExample(Protocol): + # This style is *not recommended*, but may be needed to reuse + # some complex callable types. Otherwise use regular methods. + foo: ClassVar[Callable[[object], int]] + # This may be needed to mimic descriptor access on Type[...] types, + # otherwise use a plain "bar: int" style. + bar: ClassVar[Integer] + + ex2: OtherExample + reveal_type(ex2.foo) # Revealed type is Callable[[], int] + reveal_type(ex2.bar) # Revealed type is int + .. _predefined_protocols_reference: Predefined protocol reference diff --git a/mypy/subtypes.py b/mypy/subtypes.py index acb41609fdc5..a5e6938615e7 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1457,7 +1457,8 @@ def get_member_flags(name: str, itype: Instance, class_obj: bool = False) -> set flags = {IS_VAR} if not v.is_final: flags.add(IS_SETTABLE) - if v.is_classvar: + # TODO: define cleaner rules for class vs instance variables. + if v.is_classvar and not is_descriptor(v.type): flags.add(IS_CLASSVAR) if class_obj and v.is_inferred: flags.add(IS_CLASSVAR) @@ -1465,6 +1466,15 @@ def get_member_flags(name: str, itype: Instance, class_obj: bool = False) -> set return set() +def is_descriptor(typ: Type | None) -> bool: + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return typ.type.get("__get__") is not None + if isinstance(typ, UnionType): + return all(is_descriptor(item) for item in typ.relevant_items()) + return False + + def find_node_type( node: Var | FuncBase, itype: Instance, diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index f330aa4ecc02..c6c2c5f8da98 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -4602,3 +4602,47 @@ def deco(fn: Callable[[], T]) -> Callable[[], list[T]]: ... @deco def defer() -> int: ... [builtins fixtures/list.pyi] + +[case testProtocolClassValDescriptor] +from typing import Any, Protocol, overload, ClassVar, Type + +class Desc: + @overload + def __get__(self, instance: None, owner: object) -> Desc: ... + @overload + def __get__(self, instance: object, owner: object) -> int: ... + def __get__(self, instance, owner): + pass + +class P(Protocol): + x: ClassVar[Desc] + +class C: + x = Desc() + +t: P = C() +reveal_type(t.x) # N: Revealed type is "builtins.int" +tt: Type[P] = C +reveal_type(tt.x) # N: Revealed type is "__main__.Desc" + +bad: P = C # E: Incompatible types in assignment (expression has type "type[C]", variable has type "P") \ + # N: Following member(s) of "C" have conflicts: \ + # N: x: expected "int", got "Desc" + +[case testProtocolClassValCallable] +from typing import Any, Protocol, overload, ClassVar, Type, Callable + +class P(Protocol): + foo: Callable[[object], int] + bar: ClassVar[Callable[[object], int]] + +class C: + foo: Callable[[object], int] + bar: ClassVar[Callable[[object], int]] + +t: P = C() +reveal_type(t.foo) # N: Revealed type is "def (builtins.object) -> builtins.int" +reveal_type(t.bar) # N: Revealed type is "def () -> builtins.int" +tt: Type[P] = C +reveal_type(tt.foo) # N: Revealed type is "def (builtins.object) -> builtins.int" +reveal_type(tt.bar) # N: Revealed type is "def (builtins.object) -> builtins.int" From a4801f928aaadb19f9893fe45af8e69ab6b509d0 Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Sat, 21 Jun 2025 01:08:46 +0100 Subject: [PATCH 02/15] Type ignore comments erroneously marked as unused by dmypy (#15043) There is currently a misbehaviour where "type: ignore" comments are erroneously marked as unused in re-runs of dmypy. There are also cases where errors disappear on the re-run. As far as I can tell, this only happens in modules which contain an import that we don't know how to type (such as a module which does not exist), and a submodule which is unused. There was a lot of commenting and investigation on this PR, but I hope that the committed tests and fixes illustrate and address the issue. Related to https://github.com/python/mypy/issues/9655 --------- Co-authored-by: David Seddon Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/errors.py | 4 + mypy/server/update.py | 2 + test-data/unit/daemon.test | 137 +++++++++++++++++++++++++++++++ test-data/unit/fine-grained.test | 24 ++++++ 4 files changed, 167 insertions(+) diff --git a/mypy/errors.py b/mypy/errors.py index 7a173f16d196..22a5b4ce4816 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -786,6 +786,8 @@ def generate_unused_ignore_errors(self, file: str) -> None: code=codes.UNUSED_IGNORE, blocker=False, only_once=False, + origin=(self.file, [line]), + target=self.target_module, ) self._add_error_info(file, info) @@ -837,6 +839,8 @@ def generate_ignore_without_code_errors( code=codes.IGNORE_WITHOUT_CODE, blocker=False, only_once=False, + origin=(self.file, [line]), + target=self.target_module, ) self._add_error_info(file, info) diff --git a/mypy/server/update.py b/mypy/server/update.py index 9891e2417b94..ea336154ae56 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -668,6 +668,8 @@ def restore(ids: list[str]) -> None: state.type_check_first_pass() state.type_check_second_pass() state.detect_possibly_undefined_vars() + state.generate_unused_ignore_notes() + state.generate_ignore_without_code_notes() t2 = time.time() state.finish_passes() t3 = time.time() diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index ad3b51b27dfb..295eb4000d81 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -648,6 +648,143 @@ from demo.test import a [file demo/test.py] a: int +[case testUnusedTypeIgnorePreservedOnRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --warn-unused-ignores --no-error-summary --hide-error-codes +Daemon started +$ dmypy check -- bar.py +bar.py:2: error: Unused "type: ignore" comment +== Return code: 1 +$ dmypy check -- bar.py +bar.py:2: error: Unused "type: ignore" comment +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +a = 1 # type: ignore + +[case testTypeIgnoreWithoutCodePreservedOnRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --enable-error-code ignore-without-code --no-error-summary +Daemon started +$ dmypy check -- bar.py +bar.py:2: error: "type: ignore" comment without error code [ignore-without-code] +== Return code: 1 +$ dmypy check -- bar.py +bar.py:2: error: "type: ignore" comment without error code [ignore-without-code] +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +a = 1 # type: ignore + +[case testPossiblyUndefinedVarsPreservedAfterRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --enable-error-code possibly-undefined --no-error-summary +Daemon started +$ dmypy check -- bar.py +bar.py:4: error: Name "a" may be undefined [possibly-undefined] +== Return code: 1 +$ dmypy check -- bar.py +bar.py:4: error: Name "a" may be undefined [possibly-undefined] +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +if False: + a = 1 +a + +[case testUnusedTypeIgnorePreservedOnRerunWithIgnoredMissingImports] +$ dmypy start -- --no-error-summary --ignore-missing-imports --warn-unused-ignores +Daemon started +$ dmypy check foo +foo/main.py:3: error: Unused "type: ignore" comment [unused-ignore] +== Return code: 1 +$ dmypy check foo +foo/main.py:3: error: Unused "type: ignore" comment [unused-ignore] +== Return code: 1 + +[file unused/__init__.py] +[file unused/submodule.py] +[file foo/empty.py] +[file foo/__init__.py] +from foo.main import * +from unused.submodule import * +[file foo/main.py] +from foo import empty +from foo.does_not_exist import * +a = 1 # type: ignore + +[case testModuleDoesNotExistPreservedOnRerun] +$ dmypy start -- --no-error-summary --ignore-missing-imports +Daemon started +$ dmypy check foo +foo/main.py:1: error: Module "foo" has no attribute "does_not_exist" [attr-defined] +== Return code: 1 +$ dmypy check foo +foo/main.py:1: error: Module "foo" has no attribute "does_not_exist" [attr-defined] +== Return code: 1 + +[file unused/__init__.py] +[file unused/submodule.py] +[file foo/__init__.py] +from foo.main import * +[file foo/main.py] +from foo import does_not_exist +from unused.submodule import * + +[case testReturnTypeIgnoreAfterUnknownImport] +-- Return type ignores after unknown imports and unused modules are respected on the second pass. +$ dmypy start -- --warn-unused-ignores --no-error-summary +Daemon started +$ dmypy check -- foo.py +foo.py:2: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 +$ dmypy check -- foo.py +foo.py:2: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 + +[file unused/__init__.py] +[file unused/empty.py] +[file foo.py] +from unused.empty import * +import a_module_which_does_not_exist +def is_foo() -> str: + return True # type: ignore + +[case testAttrsTypeIgnoreAfterUnknownImport] +$ dmypy start -- --warn-unused-ignores --no-error-summary +Daemon started +$ dmypy check -- foo.py +foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 +$ dmypy check -- foo.py +foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 + +[file unused/__init__.py] +[file unused/empty.py] +[file foo.py] +import attr +from unused.empty import * +import a_module_which_does_not_exist + +@attr.frozen +class A: + def __init__(self) -> None: + self.__attrs_init__() # type: ignore[attr-defined] + [case testDaemonImportAncestors] $ dmypy run test.py Daemon started diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 7e34a2352dd6..222e38ea0280 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10540,6 +10540,30 @@ from pkg.sub import modb [out] == +[case testUnusedTypeIgnorePreservedAfterChange] +# flags: --warn-unused-ignores --no-error-summary +[file main.py] +a = 1 # type: ignore +[file main.py.2] +a = 1 # type: ignore +# Comment to trigger reload. +[out] +main.py:1: error: Unused "type: ignore" comment +== +main.py:1: error: Unused "type: ignore" comment + +[case testTypeIgnoreWithoutCodePreservedAfterChange] +# flags: --enable-error-code ignore-without-code --no-error-summary +[file main.py] +a = 1 # type: ignore +[file main.py.2] +a = 1 # type: ignore +# Comment to trigger reload. +[out] +main.py:1: error: "type: ignore" comment without error code +== +main.py:1: error: "type: ignore" comment without error code + [case testFineGrainedFunctoolsPartial] import m From 934ec50744c766522329c604c6908a6ed05affd6 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Mon, 7 Jul 2025 03:07:35 -0500 Subject: [PATCH 03/15] Lessen dmypy suggest path limitations for Windows machines (#19337) In this pull request, we allow dmypy suggest absolute paths to contain the drive letter colon for Windows machines. Fixes #19335. This is done by changing how `find_node` works slightly, allowing there to be at most two colon (`:`) characters in the passed key for windows machines instead of just one like on all other platforms, and then using `rsplit` and a split limit of 1 instead of just `split` like prior. --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/suggestions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/suggestions.py b/mypy/suggestions.py index a662dd7b98e9..0c6c887d82b5 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -27,6 +27,7 @@ import itertools import json import os +import sys from collections.abc import Iterator from contextlib import contextmanager from typing import Callable, NamedTuple, TypedDict, TypeVar, cast @@ -537,12 +538,17 @@ def find_node(self, key: str) -> tuple[str, str, FuncDef]: # TODO: Also return OverloadedFuncDef -- currently these are ignored. node: SymbolNode | None = None if ":" in key: - if key.count(":") > 1: + # A colon might be part of a drive name on Windows (like `C:/foo/bar`) + # and is also used as a delimiter between file path and lineno. + # If a colon is there for any of those reasons, it must be a file+line + # reference. + platform_key_count = 2 if sys.platform == "win32" else 1 + if key.count(":") > platform_key_count: raise SuggestionFailure( "Malformed location for function: {}. Must be either" " package.module.Class.method or path/to/file.py:line".format(key) ) - file, line = key.split(":") + file, line = key.rsplit(":", 1) if not line.isdigit(): raise SuggestionFailure(f"Line number must be a number. Got {line}") line_number = int(line) From 5c65e330b0e4a188d68c04715a90e1f7d9c18df6 Mon Sep 17 00:00:00 2001 From: Chainfire Date: Wed, 2 Jul 2025 13:38:18 +0200 Subject: [PATCH 04/15] [mypyc] Fix AttributeError in async try/finally with mixed return paths (#19361) Async functions with try/finally blocks were raising AttributeError when: * Some paths in the try block return while others don't * The non-return path is executed at runtime * No further await calls are needed This occurred because mypyc's IR requires all control flow paths to assign to spill targets. The non-return path assigns NULL to maintain this invariant, but reading NULL attributes raises AttributeError in Python. Modified the GetAttr IR operation to support reading NULL attributes without raising AttributeError through a new allow_null parameter. This parameter is used specifically in try/finally resolution when reading spill targets. * Added allow_null: bool = False parameter to GetAttr.init in mypyc/ir/ops.py * When allow_null=True, sets error_kind=ERR_NEVER to prevent AttributeError * Modified read_nullable_attr in IRBuilder to create GetAttr with allow_null=True * Modified try_finally_resolve_control in statement.py to use read_nullable_attr only for spill targets (attributes starting with 'mypyc_temp') * Updated C code generation in emitfunc.py: * visit_get_attr checks for allow_null and delegates to get_attr_with_allow_null * get_attr_with_allow_null reads attributes without NULL checks and only increments reference count if not NULL Design decisions: * Targeted fix: Only applied to spill targets in try/finally resolution, not a general replacement for GetAttr. This minimizes risk and maintains existing behavior for all other attribute access. * No initialization changes: Initially tried initializing spill targets to Py_None instead of NULL, but this would incorrectly make try/finally blocks return None instead of falling through to subsequent code. Added two test cases to mypyc/test-data/run-async.test: * testAsyncTryFinallyMixedReturn: Tests the basic issue with async try/finally blocks containing mixed return/non-return paths. * testAsyncWithMixedReturn: Tests async with statements (which use try/finally under the hood) to ensure the fix works for this common pattern as well. Both tests verify that the AttributeError no longer occurs when taking the non-return path through the try block. See https://github.com/mypyc/mypyc/issues/1115 --- mypyc/codegen/emitfunc.py | 21 +++ mypyc/ir/ops.py | 9 +- mypyc/irbuild/builder.py | 9 + mypyc/irbuild/statement.py | 10 +- mypyc/test-data/run-async.test | 303 +++++++++++++++++++++++++++++++++ 5 files changed, 348 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index c854516825af..00c7fd56b899 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -358,6 +358,9 @@ def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> st return f"({cast}{obj})->{self.emitter.attr(op.attr)}" def visit_get_attr(self, op: GetAttr) -> None: + if op.allow_null: + self.get_attr_with_allow_null(op) + return dest = self.reg(op) obj = self.reg(op.obj) rtype = op.class_type @@ -426,6 +429,24 @@ def visit_get_attr(self, op: GetAttr) -> None: elif not always_defined: self.emitter.emit_line("}") + def get_attr_with_allow_null(self, op: GetAttr) -> None: + """Handle GetAttr with allow_null=True which allows NULL without raising AttributeError.""" + dest = self.reg(op) + obj = self.reg(op.obj) + rtype = op.class_type + cl = rtype.class_ir + attr_rtype, decl_cl = cl.attr_details(op.attr) + + # Direct struct access without NULL check + attr_expr = self.get_attr_expr(obj, op, decl_cl) + self.emitter.emit_line(f"{dest} = {attr_expr};") + + # Only emit inc_ref if not NULL + if attr_rtype.is_refcounted and not op.is_borrowed: + self.emitter.emit_line(f"if ({dest} != NULL) {{") + self.emitter.emit_inc_ref(dest, attr_rtype) + self.emitter.emit_line("}") + def next_branch(self) -> Branch | None: if self.op_index + 1 < len(self.ops): next_op = self.ops[self.op_index + 1] diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index eec9c34a965e..9dde658231d8 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -777,15 +777,20 @@ class GetAttr(RegisterOp): error_kind = ERR_MAGIC - def __init__(self, obj: Value, attr: str, line: int, *, borrow: bool = False) -> None: + def __init__( + self, obj: Value, attr: str, line: int, *, borrow: bool = False, allow_null: bool = False + ) -> None: super().__init__(line) self.obj = obj self.attr = attr + self.allow_null = allow_null assert isinstance(obj.type, RInstance), "Attribute access not supported: %s" % obj.type self.class_type = obj.type attr_type = obj.type.attr_type(attr) self.type = attr_type - if attr_type.error_overlap: + if allow_null: + self.error_kind = ERR_NEVER + elif attr_type.error_overlap: self.error_kind = ERR_MAGIC_OVERLAPPING self.is_borrowed = borrow and attr_type.is_refcounted diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 75e059a5b570..878c5e76df3d 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -708,6 +708,15 @@ def read( assert False, "Unsupported lvalue: %r" % target + def read_nullable_attr(self, obj: Value, attr: str, line: int = -1) -> Value: + """Read an attribute that might be NULL without raising AttributeError. + + This is used for reading spill targets in try/finally blocks where NULL + indicates the non-return path was taken. + """ + assert isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class + return self.add(GetAttr(obj, attr, line, allow_null=True)) + def assign(self, target: Register | AssignmentTarget, rvalue_reg: Value, line: int) -> None: if isinstance(target, Register): self.add(Assign(target, self.coerce_rvalue(rvalue_reg, target.type, line))) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 16a0483a8729..5c32d8f1a50c 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -46,6 +46,7 @@ YieldExpr, YieldFromExpr, ) +from mypyc.common import TEMP_ATTR_NAME from mypyc.ir.ops import ( NAMESPACE_MODULE, NO_TRACEBACK_LINE_NO, @@ -653,10 +654,15 @@ def try_finally_resolve_control( if ret_reg: builder.activate_block(rest) return_block, rest = BasicBlock(), BasicBlock() - builder.add(Branch(builder.read(ret_reg), rest, return_block, Branch.IS_ERROR)) + # For spill targets in try/finally, use nullable read to avoid AttributeError + if isinstance(ret_reg, AssignmentTargetAttr) and ret_reg.attr.startswith(TEMP_ATTR_NAME): + ret_val = builder.read_nullable_attr(ret_reg.obj, ret_reg.attr, -1) + else: + ret_val = builder.read(ret_reg) + builder.add(Branch(ret_val, rest, return_block, Branch.IS_ERROR)) builder.activate_block(return_block) - builder.nonlocal_control[-1].gen_return(builder, builder.read(ret_reg), -1) + builder.nonlocal_control[-1].gen_return(builder, ret_val, -1) # TODO: handle break/continue builder.activate_block(rest) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 11ce67077270..2dad720f99cd 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -643,3 +643,306 @@ def test_async_def_contains_two_nested_functions() -> None: [file asyncio/__init__.pyi] def run(x: object) -> object: ... + +[case testAsyncTryFinallyMixedReturn] +# This used to raise an AttributeError, when: +# - the try block contains multiple paths +# - at least one of those explicitly returns +# - at least one of those does not explicitly return +# - the non-returning path is taken at runtime + +import asyncio + + +async def test_mixed_return(b: bool) -> bool: + try: + if b: + return b + finally: + pass + return b + + +async def test_run() -> None: + # Test return path + result1 = await test_mixed_return(True) + assert result1 == True + + # Test non-return path + result2 = await test_mixed_return(False) + assert result2 == False + + +def test_async_try_finally_mixed_return() -> None: + asyncio.run(test_run()) + +[file driver.py] +from native import test_async_try_finally_mixed_return +test_async_try_finally_mixed_return() + +[file asyncio/__init__.pyi] +def run(x: object) -> object: ... + +[case testAsyncWithMixedReturn] +# This used to raise an AttributeError, related to +# testAsyncTryFinallyMixedReturn, this is essentially +# a far more extensive version of that test surfacing +# more edge cases + +import asyncio +from typing import Optional, Type, Literal + + +class AsyncContextManager: + async def __aenter__(self) -> "AsyncContextManager": + return self + + async def __aexit__( + self, + t: Optional[Type[BaseException]], + v: Optional[BaseException], + tb: object, + ) -> Literal[False]: + return False + + +# Simple async functions (generator class) +async def test_gen_1(b: bool) -> bool: + async with AsyncContextManager(): + if b: + return b + return b + + +async def test_gen_2(b: bool) -> bool: + async with AsyncContextManager(): + if b: + return b + else: + return b + + +async def test_gen_3(b: bool) -> bool: + async with AsyncContextManager(): + if b: + return b + else: + pass + return b + + +async def test_gen_4(b: bool) -> bool: + ret: bool + async with AsyncContextManager(): + if b: + ret = b + else: + ret = b + return ret + + +async def test_gen_5(i: int) -> int: + async with AsyncContextManager(): + if i == 1: + return i + elif i == 2: + pass + elif i == 3: + return i + return i + + +async def test_gen_6(i: int) -> int: + async with AsyncContextManager(): + if i == 1: + return i + elif i == 2: + return i + elif i == 3: + return i + return i + + +async def test_gen_7(i: int) -> int: + async with AsyncContextManager(): + if i == 1: + return i + elif i == 2: + return i + elif i == 3: + return i + else: + return i + + +# Async functions with nested functions (environment class) +async def test_env_1(b: bool) -> bool: + def helper() -> bool: + return True + + async with AsyncContextManager(): + if b: + return helper() + return b + + +async def test_env_2(b: bool) -> bool: + def helper() -> bool: + return True + + async with AsyncContextManager(): + if b: + return helper() + else: + return b + + +async def test_env_3(b: bool) -> bool: + def helper() -> bool: + return True + + async with AsyncContextManager(): + if b: + return helper() + else: + pass + return b + + +async def test_env_4(b: bool) -> bool: + def helper() -> bool: + return True + + ret: bool + async with AsyncContextManager(): + if b: + ret = helper() + else: + ret = b + return ret + + +async def test_env_5(i: int) -> int: + def helper() -> int: + return 1 + + async with AsyncContextManager(): + if i == 1: + return helper() + elif i == 2: + pass + elif i == 3: + return i + return i + + +async def test_env_6(i: int) -> int: + def helper() -> int: + return 1 + + async with AsyncContextManager(): + if i == 1: + return helper() + elif i == 2: + return i + elif i == 3: + return i + return i + + +async def test_env_7(i: int) -> int: + def helper() -> int: + return 1 + + async with AsyncContextManager(): + if i == 1: + return helper() + elif i == 2: + return i + elif i == 3: + return i + else: + return i + + +async def run_all_tests() -> None: + # Test simple async functions (generator class) + # test_env_1: mixed return/no-return + assert await test_gen_1(True) is True + assert await test_gen_1(False) is False + + # test_gen_2: all branches return + assert await test_gen_2(True) is True + assert await test_gen_2(False) is False + + # test_gen_3: mixed return/pass + assert await test_gen_3(True) is True + assert await test_gen_3(False) is False + + # test_gen_4: no returns in async with + assert await test_gen_4(True) is True + assert await test_gen_4(False) is False + + # test_gen_5: multiple branches, some return + assert await test_gen_5(0) == 0 + assert await test_gen_5(1) == 1 + assert await test_gen_5(2) == 2 + assert await test_gen_5(3) == 3 + + # test_gen_6: all explicit branches return, implicit fallthrough + assert await test_gen_6(0) == 0 + assert await test_gen_6(1) == 1 + assert await test_gen_6(2) == 2 + assert await test_gen_6(3) == 3 + + # test_gen_7: all branches return including else + assert await test_gen_7(0) == 0 + assert await test_gen_7(1) == 1 + assert await test_gen_7(2) == 2 + assert await test_gen_7(3) == 3 + + # Test async functions with nested functions (environment class) + # test_env_1: mixed return/no-return + assert await test_env_1(True) is True + assert await test_env_1(False) is False + + # test_env_2: all branches return + assert await test_env_2(True) is True + assert await test_env_2(False) is False + + # test_env_3: mixed return/pass + assert await test_env_3(True) is True + assert await test_env_3(False) is False + + # test_env_4: no returns in async with + assert await test_env_4(True) is True + assert await test_env_4(False) is False + + # test_env_5: multiple branches, some return + assert await test_env_5(0) == 0 + assert await test_env_5(1) == 1 + assert await test_env_5(2) == 2 + assert await test_env_5(3) == 3 + + # test_env_6: all explicit branches return, implicit fallthrough + assert await test_env_6(0) == 0 + assert await test_env_6(1) == 1 + assert await test_env_6(2) == 2 + assert await test_env_6(3) == 3 + + # test_env_7: all branches return including else + assert await test_env_7(0) == 0 + assert await test_env_7(1) == 1 + assert await test_env_7(2) == 2 + assert await test_env_7(3) == 3 + + +def test_async_with_mixed_return() -> None: + asyncio.run(run_all_tests()) + +[file driver.py] +from native import test_async_with_mixed_return +test_async_with_mixed_return() + +[file asyncio/__init__.pyi] +def run(x: object) -> object: ... From 09ba1f6488b3e8d91c5204839421c61c306ff252 Mon Sep 17 00:00:00 2001 From: Chainfire Date: Thu, 10 Jul 2025 11:58:57 +0200 Subject: [PATCH 05/15] [mypyc] Fix exception swallowing in async try/finally blocks with await (#19353) When a try/finally block in an async function contains an await statement in the finally block, exceptions raised in the try block are silently swallowed if a context switch occurs. This happens because mypyc stores exception information in registers that don't survive across await points. The Problem: - mypyc's transform_try_finally_stmt uses error_catch_op to save exceptions - to a register, then reraise_exception_op to restore from that register - When await causes a context switch, register values are lost - The exception information is gone, causing silent exception swallowing The Solution: - Add new transform_try_finally_stmt_async for async-aware exception handling - Use sys.exc_info() to preserve exceptions across context switches instead - of registers - Check error indicator first to handle new exceptions raised in finally - Route to async version when finally block contains await expressions Implementation Details: - transform_try_finally_stmt_async uses get_exc_info_op/restore_exc_info_op - which work with sys.exc_info() that survives context switches - Proper exception priority: new exceptions in finally replace originals - Added has_await_in_block helper to detect await expressions Test Coverage: Added comprehensive async exception handling tests: - testAsyncTryExceptFinallyAwait: 8 test cases covering various scenarios - Simple try/finally with exception and await - Exception caught but not re-raised - Exception caught and re-raised - Different exception raised in except - Try/except inside finally block - Try/finally inside finally block - Control case without await - Normal flow without exceptions - testAsyncContextManagerExceptionHandling: Verifies async with still works - Basic exception propagation - Exception in **aexit** replacing original See mypyc/mypyc#1114. --- mypyc/irbuild/statement.py | 137 ++++++++++++++++++++- mypyc/test-data/run-async.test | 211 +++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 5c32d8f1a50c..f780db2249df 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -12,6 +12,7 @@ from collections.abc import Sequence from typing import Callable +import mypy.nodes from mypy.nodes import ( ARG_NAMED, ARG_POS, @@ -101,6 +102,7 @@ get_exc_info_op, get_exc_value_op, keep_propagating_op, + no_err_occurred_op, raise_exception_op, reraise_exception_op, restore_exc_info_op, @@ -679,7 +681,7 @@ def try_finally_resolve_control( def transform_try_finally_stmt( - builder: IRBuilder, try_body: GenFunc, finally_body: GenFunc + builder: IRBuilder, try_body: GenFunc, finally_body: GenFunc, line: int = -1 ) -> None: """Generalized try/finally handling that takes functions to gen the bodies. @@ -715,6 +717,118 @@ def transform_try_finally_stmt( builder.activate_block(out_block) +def transform_try_finally_stmt_async( + builder: IRBuilder, try_body: GenFunc, finally_body: GenFunc, line: int = -1 +) -> None: + """Async-aware try/finally handling for when finally contains await. + + This version uses a modified approach that preserves exceptions across await.""" + + # We need to handle returns properly, so we'll use TryFinallyNonlocalControl + # to track return values, similar to the regular try/finally implementation + + err_handler, main_entry, return_entry, finally_entry = ( + BasicBlock(), + BasicBlock(), + BasicBlock(), + BasicBlock(), + ) + + # Track if we're returning from the try block + control = TryFinallyNonlocalControl(return_entry) + builder.builder.push_error_handler(err_handler) + builder.nonlocal_control.append(control) + builder.goto_and_activate(BasicBlock()) + try_body() + builder.goto(main_entry) + builder.nonlocal_control.pop() + builder.builder.pop_error_handler() + ret_reg = control.ret_reg + + # Normal case - no exception or return + builder.activate_block(main_entry) + builder.goto(finally_entry) + + # Return case + builder.activate_block(return_entry) + builder.goto(finally_entry) + + # Exception case - need to catch to clear the error indicator + builder.activate_block(err_handler) + # Catch the error to clear Python's error indicator + builder.call_c(error_catch_op, [], line) + # We're not going to use old_exc since it won't survive await + # The exception is now in sys.exc_info() + builder.goto(finally_entry) + + # Finally block + builder.activate_block(finally_entry) + + # Execute finally body + finally_body() + + # After finally, we need to handle exceptions carefully: + # 1. If finally raised a new exception, it's in the error indicator - let it propagate + # 2. If finally didn't raise, check if we need to reraise the original from sys.exc_info() + # 3. If there was a return, return that value + # 4. Otherwise, normal exit + + # First, check if there's a current exception in the error indicator + # (this would be from the finally block) + no_current_exc = builder.call_c(no_err_occurred_op, [], line) + finally_raised = BasicBlock() + check_original = BasicBlock() + builder.add(Branch(no_current_exc, check_original, finally_raised, Branch.BOOL)) + + # Finally raised an exception - let it propagate naturally + builder.activate_block(finally_raised) + builder.call_c(keep_propagating_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + # No exception from finally, check if we need to handle return or original exception + builder.activate_block(check_original) + + # Check if we have a return value + if ret_reg: + return_block, check_old_exc = BasicBlock(), BasicBlock() + builder.add(Branch(builder.read(ret_reg), check_old_exc, return_block, Branch.IS_ERROR)) + + builder.activate_block(return_block) + builder.nonlocal_control[-1].gen_return(builder, builder.read(ret_reg), -1) + + builder.activate_block(check_old_exc) + + # Check if we need to reraise the original exception from sys.exc_info + exc_info = builder.call_c(get_exc_info_op, [], line) + exc_type = builder.add(TupleGet(exc_info, 0, line)) + + # Check if exc_type is None + none_obj = builder.none_object() + has_exc = builder.binary_op(exc_type, none_obj, "is not", line) + + reraise_block, exit_block = BasicBlock(), BasicBlock() + builder.add(Branch(has_exc, reraise_block, exit_block, Branch.BOOL)) + + # Reraise the original exception + builder.activate_block(reraise_block) + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + + # Normal exit + builder.activate_block(exit_block) + + +# A simple visitor to detect await expressions +class AwaitDetector(mypy.traverser.TraverserVisitor): + def __init__(self) -> None: + super().__init__() + self.has_await = False + + def visit_await_expr(self, o: mypy.nodes.AwaitExpr) -> None: + self.has_await = True + super().visit_await_expr(o) + + def transform_try_stmt(builder: IRBuilder, t: TryStmt) -> None: # Our compilation strategy for try/except/else/finally is to # treat try/except/else and try/finally as separate language @@ -723,6 +837,17 @@ def transform_try_stmt(builder: IRBuilder, t: TryStmt) -> None: # body of a try/finally block. if t.is_star: builder.error("Exception groups and except* cannot be compiled yet", t.line) + + # Check if we're in an async function with a finally block that contains await + use_async_version = False + if t.finally_body and builder.fn_info.is_coroutine: + detector = AwaitDetector() + t.finally_body.accept(detector) + + if detector.has_await: + # Use the async version that handles exceptions correctly + use_async_version = True + if t.finally_body: def transform_try_body() -> None: @@ -733,7 +858,14 @@ def transform_try_body() -> None: body = t.finally_body - transform_try_finally_stmt(builder, transform_try_body, lambda: builder.accept(body)) + if use_async_version: + transform_try_finally_stmt_async( + builder, transform_try_body, lambda: builder.accept(body), t.line + ) + else: + transform_try_finally_stmt( + builder, transform_try_body, lambda: builder.accept(body), t.line + ) else: transform_try_except_stmt(builder, t) @@ -824,6 +956,7 @@ def finally_body() -> None: builder, lambda: transform_try_except(builder, try_body, [(None, None, except_body)], None, line), finally_body, + line, ) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 2dad720f99cd..d1fb68d9f013 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -946,3 +946,214 @@ test_async_with_mixed_return() [file asyncio/__init__.pyi] def run(x: object) -> object: ... + +[case testAsyncTryExceptFinallyAwait] +import asyncio +from testutil import assertRaises + +class TestError(Exception): + pass + +# Test 0: Simplest case - just try/finally with raise and await +async def simple_try_finally_await() -> None: + try: + raise ValueError("simple error") + finally: + await asyncio.sleep(0) + +# Test 1: Raise inside try, catch in except, don't re-raise +async def async_try_except_no_reraise() -> int: + try: + raise ValueError("test error") + return 1 # Never reached + except ValueError: + return 2 # Should return this + finally: + await asyncio.sleep(0) + return 3 # Should not reach this + +# Test 2: Raise inside try, catch in except, re-raise +async def async_try_except_reraise() -> int: + try: + raise ValueError("test error") + return 1 # Never reached + except ValueError: + raise # Re-raise the exception + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +# Test 3: Raise inside try, catch in except, raise different error +async def async_try_except_raise_different() -> int: + try: + raise ValueError("original error") + return 1 # Never reached + except ValueError: + raise RuntimeError("different error") + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +# Test 4: Another try/except block inside finally +async def async_try_except_inside_finally() -> int: + try: + raise ValueError("outer error") + return 1 # Never reached + finally: + await asyncio.sleep(0) + try: + raise RuntimeError("inner error") + except RuntimeError: + pass # Catch inner error + return 2 # What happens after finally with inner exception handled? + +# Test 5: Another try/finally block inside finally +async def async_try_finally_inside_finally() -> int: + try: + raise ValueError("outer error") + return 1 # Never reached + finally: + await asyncio.sleep(0) + try: + raise RuntimeError("inner error") + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +# Control case: No await in finally - should work correctly +async def async_exception_no_await_in_finally() -> None: + """Control case: This works correctly - exception propagates""" + try: + raise TestError("This exception will propagate!") + finally: + pass # No await here + +# Test function with no exception to check normal flow +async def async_no_exception_with_await_in_finally() -> int: + try: + return 1 # Normal return + finally: + await asyncio.sleep(0) + return 2 # Should not reach this + +def test_async_try_except_finally_await() -> None: + # Test 0: Simplest case - just try/finally with exception + # Expected: ValueError propagates + with assertRaises(ValueError): + asyncio.run(simple_try_finally_await()) + + # Test 1: Exception caught, not re-raised + # Expected: return 2 (from except block) + result = asyncio.run(async_try_except_no_reraise()) + assert result == 2, f"Expected 2, got {result}" + + # Test 2: Exception caught and re-raised + # Expected: ValueError propagates + with assertRaises(ValueError): + asyncio.run(async_try_except_reraise()) + + # Test 3: Exception caught, different exception raised + # Expected: RuntimeError propagates + with assertRaises(RuntimeError): + asyncio.run(async_try_except_raise_different()) + + # Test 4: Try/except inside finally + # Expected: ValueError propagates (outer exception) + with assertRaises(ValueError): + asyncio.run(async_try_except_inside_finally()) + + # Test 5: Try/finally inside finally + # Expected: RuntimeError propagates (inner error) + with assertRaises(RuntimeError): + asyncio.run(async_try_finally_inside_finally()) + + # Control case: No await in finally (should work correctly) + with assertRaises(TestError): + asyncio.run(async_exception_no_await_in_finally()) + + # Test normal flow (no exception) + # Expected: return 1 + result = asyncio.run(async_no_exception_with_await_in_finally()) + assert result == 1, f"Expected 1, got {result}" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... +def run(x: object) -> object: ... + +[case testAsyncContextManagerExceptionHandling] +import asyncio +from typing import Optional, Type +from testutil import assertRaises + +# Test 1: Basic async context manager that doesn't suppress exceptions +class AsyncContextManager: + async def __aenter__(self) -> 'AsyncContextManager': + return self + + async def __aexit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: object) -> None: + # This await in __aexit__ is like await in finally + await asyncio.sleep(0) + # Don't suppress the exception (return None/False) + +async def func_with_async_context_manager() -> str: + async with AsyncContextManager(): + raise ValueError("Exception inside async with") + return "should not reach" # Never reached + return "should not reach either" # Never reached + +async def test_basic_exception() -> str: + try: + await func_with_async_context_manager() + return "func_a returned normally - bug!" + except ValueError: + return "caught ValueError - correct!" + except Exception as e: + return f"caught different exception: {type(e).__name__}" + +# Test 2: Async context manager that raises a different exception in __aexit__ +class AsyncContextManagerRaisesInExit: + async def __aenter__(self) -> 'AsyncContextManagerRaisesInExit': + return self + + async def __aexit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: object) -> None: + # This await in __aexit__ is like await in finally + await asyncio.sleep(0) + # Raise a different exception - this should replace the original exception + raise RuntimeError("Exception in __aexit__") + +async def func_with_raising_context_manager() -> str: + async with AsyncContextManagerRaisesInExit(): + raise ValueError("Original exception") + return "should not reach" # Never reached + return "should not reach either" # Never reached + +async def test_exception_in_aexit() -> str: + try: + await func_with_raising_context_manager() + return "func returned normally - unexpected!" + except RuntimeError: + return "caught RuntimeError - correct!" + except ValueError: + return "caught ValueError - original exception not replaced!" + except Exception as e: + return f"caught different exception: {type(e).__name__}" + +def test_async_context_manager_exception_handling() -> None: + # Test 1: Basic exception propagation + result = asyncio.run(test_basic_exception()) + # Expected: "caught ValueError - correct!" + assert result == "caught ValueError - correct!", f"Expected exception to propagate, got: {result}" + + # Test 2: Exception raised in __aexit__ replaces original exception + result = asyncio.run(test_exception_in_aexit()) + # Expected: "caught RuntimeError - correct!" + # (The RuntimeError from __aexit__ should replace the ValueError) + assert result == "caught RuntimeError - correct!", f"Expected RuntimeError from __aexit__, got: {result}" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... +def run(x: object) -> object: ... From ab4fd57d45b7f81cf281b17b7d3697ac9f79bc15 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Fri, 20 Jun 2025 17:48:30 +0200 Subject: [PATCH 06/15] Improve the handling of "iteration dependent" errors and notes in finally clauses. (#19270) Fixes #19269 This PR refactors the logic implemented in #19118 (which only targeted repeatedly checked loops) and applies it to repeatedly checked finally clauses. I moved nearly all relevant code to the class `LoopErrorWatcher`, which now has the more general name `IterationErrorWatcher`, to avoid code duplication. However, one duplication is left, which concerns error reporting. It would be nice and easy to move this functionality to `IterationErrorWatcher`, too, but this would result in import cycles, and I am unsure if working with `TYPE_CHECKING` and postponed importing is acceptable in such cases (both for Mypy and Mypyc). After the refactoring, it should not be much effort to apply the logic to other cases where code sections are analysed iteratively. However, the only thing that comes to my mind is the repeated checking of functions with arguments that contain constrained type variables. I will check it. If anyone finds a similar case and the solution is as simple as expected, we could add the fix to this PR, of course. --- mypy/checker.py | 67 ++++++-------- mypy/errors.py | 96 ++++++++++++++++---- test-data/unit/check-narrowing.test | 19 ++++ test-data/unit/check-redefine2.test | 3 +- test-data/unit/check-union-error-syntax.test | 21 +++++ 5 files changed, 148 insertions(+), 58 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 0639340d30bb..f929178e374e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -25,7 +25,14 @@ from mypy.constraints import SUPERTYPE_OF from mypy.erasetype import erase_type, erase_typevars, remove_instance_last_known_values from mypy.errorcodes import TYPE_VAR, UNUSED_AWAITABLE, UNUSED_COROUTINE, ErrorCode -from mypy.errors import ErrorInfo, Errors, ErrorWatcher, LoopErrorWatcher, report_internal_error +from mypy.errors import ( + ErrorInfo, + Errors, + ErrorWatcher, + IterationDependentErrors, + IterationErrorWatcher, + report_internal_error, +) from mypy.expandtype import expand_type from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash from mypy.maptype import map_instance_to_supertype @@ -598,26 +605,15 @@ def accept_loop( # on without bound otherwise) widened_old = len(self.widened_vars) - # one set of `unreachable`, `redundant-expr`, and `redundant-casts` errors - # per iteration step: - uselessness_errors = [] - # one set of unreachable line numbers per iteration step: - unreachable_lines = [] - # one set of revealed types per line where `reveal_type` is used (each - # created set can grow during the iteration): - revealed_types = defaultdict(set) + iter_errors = IterationDependentErrors() iter = 1 while True: with self.binder.frame_context(can_skip=True, break_frame=2, continue_frame=1): if on_enter_body is not None: on_enter_body() - with LoopErrorWatcher(self.msg.errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: self.accept(body) - uselessness_errors.append(watcher.uselessness_errors) - unreachable_lines.append(watcher.unreachable_lines) - for key, values in watcher.revealed_types.items(): - revealed_types[key].update(values) partials_new = sum(len(pts.map) for pts in self.partial_types) widened_new = len(self.widened_vars) @@ -639,29 +635,10 @@ def accept_loop( if iter == 20: raise RuntimeError("Too many iterations when checking a loop") - # Report only those `unreachable`, `redundant-expr`, and `redundant-casts` - # errors that could not be ruled out in any iteration step: - persistent_uselessness_errors = set() - for candidate in set(itertools.chain(*uselessness_errors)): - if all( - (candidate in errors) or (candidate[2] in lines) - for errors, lines in zip(uselessness_errors, unreachable_lines) - ): - persistent_uselessness_errors.add(candidate) - for error_info in persistent_uselessness_errors: - context = Context(line=error_info[2], column=error_info[3]) - context.end_line = error_info[4] - context.end_column = error_info[5] - self.msg.fail(error_info[1], context, code=error_info[0]) - - # Report all types revealed in at least one iteration step: - for note_info, types in revealed_types.items(): - sorted_ = sorted(types, key=lambda typ: typ.lower()) - revealed = sorted_[0] if len(types) == 1 else f"Union[{', '.join(sorted_)}]" - context = Context(line=note_info[1], column=note_info[2]) - context.end_line = note_info[3] - context.end_column = note_info[4] - self.note(f'Revealed type is "{revealed}"', context) + for error_info in watcher.yield_error_infos(): + self.msg.fail(*error_info[:2], code=error_info[2]) + for note_info in watcher.yield_note_infos(self.options): + self.note(*note_info) # If exit_condition is set, assume it must be False on exit from the loop: if exit_condition: @@ -4948,6 +4925,9 @@ def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) def visit_try_stmt(self, s: TryStmt) -> None: """Type check a try statement.""" + + iter_errors = None + # Our enclosing frame will get the result if the try/except falls through. # This one gets all possible states after the try block exited abnormally # (by exception, return, break, etc.) @@ -4962,7 +4942,9 @@ def visit_try_stmt(self, s: TryStmt) -> None: self.visit_try_without_finally(s, try_frame=bool(s.finally_body)) if s.finally_body: # First we check finally_body is type safe on all abnormal exit paths - self.accept(s.finally_body) + iter_errors = IterationDependentErrors() + with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + self.accept(s.finally_body) if s.finally_body: # Then we try again for the more restricted set of options @@ -4976,8 +4958,15 @@ def visit_try_stmt(self, s: TryStmt) -> None: # type checks in both contexts, but only the resulting types # from the latter context affect the type state in the code # that follows the try statement.) + assert iter_errors is not None if not self.binder.is_unreachable(): - self.accept(s.finally_body) + with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + self.accept(s.finally_body) + + for error_info in watcher.yield_error_infos(): + self.msg.fail(*error_info[:2], code=error_info[2]) + for note_info in watcher.yield_note_infos(self.options): + self.msg.note(*note_info) def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: """Type check a try statement, ignoring the finally block. diff --git a/mypy/errors.py b/mypy/errors.py index 22a5b4ce4816..5dd411c39e95 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -4,13 +4,15 @@ import sys import traceback from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Iterable, Iterator +from itertools import chain from typing import Callable, Final, NoReturn, Optional, TextIO, TypeVar from typing_extensions import Literal, Self, TypeAlias as _TypeAlias from mypy import errorcodes as codes from mypy.error_formatter import ErrorFormatter from mypy.errorcodes import IMPORT, IMPORT_NOT_FOUND, IMPORT_UNTYPED, ErrorCode, mypy_error_codes +from mypy.nodes import Context from mypy.options import Options from mypy.scope import Scope from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file @@ -219,23 +221,43 @@ def filtered_errors(self) -> list[ErrorInfo]: return self._filtered -class LoopErrorWatcher(ErrorWatcher): - """Error watcher that filters and separately collects `unreachable` errors, - `redundant-expr` and `redundant-casts` errors, and revealed types when analysing - loops iteratively to help avoid making too-hasty reports.""" +class IterationDependentErrors: + """An `IterationDependentErrors` instance serves to collect the `unreachable`, + `redundant-expr`, and `redundant-casts` errors, as well as the revealed types, + handled by the individual `IterationErrorWatcher` instances sequentially applied to + the same code section.""" - # Meaning of the tuple items: ErrorCode, message, line, column, end_line, end_column: - uselessness_errors: set[tuple[ErrorCode, str, int, int, int, int]] + # One set of `unreachable`, `redundant-expr`, and `redundant-casts` errors per + # iteration step. Meaning of the tuple items: ErrorCode, message, line, column, + # end_line, end_column. + uselessness_errors: list[set[tuple[ErrorCode, str, int, int, int, int]]] - # Meaning of the tuple items: function_or_member, line, column, end_line, end_column: + # One set of unreachable line numbers per iteration step. Not only the lines where + # the error report occurs but really all unreachable lines. + unreachable_lines: list[set[int]] + + # One set of revealed types for each `reveal_type` statement. Each created set can + # grow during the iteration. Meaning of the tuple items: function_or_member, line, + # column, end_line, end_column: revealed_types: dict[tuple[str | None, int, int, int, int], set[str]] - # Not only the lines where the error report occurs but really all unreachable lines: - unreachable_lines: set[int] + def __init__(self) -> None: + self.uselessness_errors = [] + self.unreachable_lines = [] + self.revealed_types = defaultdict(set) + + +class IterationErrorWatcher(ErrorWatcher): + """Error watcher that filters and separately collects `unreachable` errors, + `redundant-expr` and `redundant-casts` errors, and revealed types when analysing + code sections iteratively to help avoid making too-hasty reports.""" + + iteration_dependent_errors: IterationDependentErrors def __init__( self, errors: Errors, + iteration_dependent_errors: IterationDependentErrors, *, filter_errors: bool | Callable[[str, ErrorInfo], bool] = False, save_filtered_errors: bool = False, @@ -247,31 +269,71 @@ def __init__( save_filtered_errors=save_filtered_errors, filter_deprecated=filter_deprecated, ) - self.uselessness_errors = set() - self.unreachable_lines = set() - self.revealed_types = defaultdict(set) + self.iteration_dependent_errors = iteration_dependent_errors + iteration_dependent_errors.uselessness_errors.append(set()) + iteration_dependent_errors.unreachable_lines.append(set()) def on_error(self, file: str, info: ErrorInfo) -> bool: + """Filter out the "iteration-dependent" errors and notes and store their + information to handle them after iteration is completed.""" + + iter_errors = self.iteration_dependent_errors if info.code in (codes.UNREACHABLE, codes.REDUNDANT_EXPR, codes.REDUNDANT_CAST): - self.uselessness_errors.add( + iter_errors.uselessness_errors[-1].add( (info.code, info.message, info.line, info.column, info.end_line, info.end_column) ) if info.code == codes.UNREACHABLE: - self.unreachable_lines.update(range(info.line, info.end_line + 1)) + iter_errors.unreachable_lines[-1].update(range(info.line, info.end_line + 1)) return True if info.code == codes.MISC and info.message.startswith("Revealed type is "): key = info.function_or_member, info.line, info.column, info.end_line, info.end_column types = info.message.split('"')[1] if types.startswith("Union["): - self.revealed_types[key].update(types[6:-1].split(", ")) + iter_errors.revealed_types[key].update(types[6:-1].split(", ")) else: - self.revealed_types[key].add(types) + iter_errors.revealed_types[key].add(types) return True return super().on_error(file, info) + def yield_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]: + """Report only those `unreachable`, `redundant-expr`, and `redundant-casts` + errors that could not be ruled out in any iteration step.""" + + persistent_uselessness_errors = set() + iter_errors = self.iteration_dependent_errors + for candidate in set(chain(*iter_errors.uselessness_errors)): + if all( + (candidate in errors) or (candidate[2] in lines) + for errors, lines in zip( + iter_errors.uselessness_errors, iter_errors.unreachable_lines + ) + ): + persistent_uselessness_errors.add(candidate) + for error_info in persistent_uselessness_errors: + context = Context(line=error_info[2], column=error_info[3]) + context.end_line = error_info[4] + context.end_column = error_info[5] + yield error_info[1], context, error_info[0] + + def yield_note_infos(self, options: Options) -> Iterator[tuple[str, Context]]: + """Yield all types revealed in at least one iteration step.""" + + for note_info, types in self.iteration_dependent_errors.revealed_types.items(): + sorted_ = sorted(types, key=lambda typ: typ.lower()) + if len(types) == 1: + revealed = sorted_[0] + elif options.use_or_syntax(): + revealed = " | ".join(sorted_) + else: + revealed = f"Union[{', '.join(sorted_)}]" + context = Context(line=note_info[1], column=note_info[2]) + context.end_line = note_info[3] + context.end_column = note_info[4] + yield f'Revealed type is "{revealed}"', context + class Errors: """Container for compile errors. diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 3778c5276576..36a148fc47df 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2446,6 +2446,25 @@ while x is not None and b(): x = f() [builtins fixtures/primitives.pyi] +[case testAvoidFalseUnreachableInFinally] +# flags: --allow-redefinition-new --local-partial-types --warn-unreachable +def f() -> None: + try: + x = 1 + if int(): + x = "" + return + if int(): + x = None + return + finally: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + if isinstance(x, str): + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + +[builtins fixtures/isinstancelist.pyi] + [case testNarrowingTypeVarMultiple] from typing import TypeVar diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 1062be6976c0..924e66584669 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -791,8 +791,7 @@ def f3() -> None: x = "" return finally: - reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" \ - # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" reveal_type(x) # N: Revealed type is "builtins.int" def f4() -> None: diff --git a/test-data/unit/check-union-error-syntax.test b/test-data/unit/check-union-error-syntax.test index 3c541173a891..d41281b774e1 100644 --- a/test-data/unit/check-union-error-syntax.test +++ b/test-data/unit/check-union-error-syntax.test @@ -55,3 +55,24 @@ from typing import Literal, Union x : Union[Literal[1], None] x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", variable has type "Optional[Literal[1]]") [builtins fixtures/tuple.pyi] + +[case testUnionSyntaxRecombined] +# flags: --python-version 3.10 --force-union-syntax --allow-redefinition-new --local-partial-types +# The following revealed type is recombined because the finally body is visited twice. +try: + x = 1 + x = "" +finally: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/isinstancelist.pyi] + +[case testOrSyntaxRecombined] +# flags: --python-version 3.10 --no-force-union-syntax --allow-redefinition-new --local-partial-types +# The following revealed type is recombined because the finally body is visited twice. +# ToDo: Improve this recombination logic, especially (but not only) for the "or syntax". +try: + x = 1 + x = "" +finally: + reveal_type(x) # N: Revealed type is "builtins.int | builtins.str | builtins.str" +[builtins fixtures/isinstancelist.pyi] From a182dec997b418b925fe0c28575c50debba0bb3a Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Mon, 7 Jul 2025 11:24:42 +0200 Subject: [PATCH 07/15] Combine the revealed types of multiple iteration steps in a more robust manner. (#19324) This PR fixes a regression introduced in #19118 and discussed in #19270. The combination of the revealed types of individual iteration steps now relies on collecting the original type objects instead of parts of preliminary `revealed_type` notes. As @JukkaL suspected, this approach is much more straightforward than introducing a sufficiently complete `revealed_type` note parser. Please note that I appended a commit that refactors already existing code. It is mainly code-moving, so I hope it does not complicate the review of this PR. --- mypy/checker.py | 19 ++-- mypy/errors.py | 103 +++++++++---------- mypy/messages.py | 34 +++++- test-data/unit/check-inference.test | 4 +- test-data/unit/check-narrowing.test | 2 +- test-data/unit/check-redefine2.test | 4 +- test-data/unit/check-typevar-tuple.test | 2 +- test-data/unit/check-union-error-syntax.test | 7 +- 8 files changed, 95 insertions(+), 80 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f929178e374e..217a4a885dd8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -612,7 +612,7 @@ def accept_loop( if on_enter_body is not None: on_enter_body() - with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors): self.accept(body) partials_new = sum(len(pts.map) for pts in self.partial_types) @@ -635,10 +635,7 @@ def accept_loop( if iter == 20: raise RuntimeError("Too many iterations when checking a loop") - for error_info in watcher.yield_error_infos(): - self.msg.fail(*error_info[:2], code=error_info[2]) - for note_info in watcher.yield_note_infos(self.options): - self.note(*note_info) + self.msg.iteration_dependent_errors(iter_errors) # If exit_condition is set, assume it must be False on exit from the loop: if exit_condition: @@ -3027,7 +3024,7 @@ def is_noop_for_reachability(self, s: Statement) -> bool: if isinstance(s.expr, EllipsisExpr): return True elif isinstance(s.expr, CallExpr): - with self.expr_checker.msg.filter_errors(): + with self.expr_checker.msg.filter_errors(filter_revealed_type=True): typ = get_proper_type( self.expr_checker.accept( s.expr, allow_none_return=True, always_allow_any=True @@ -4943,7 +4940,7 @@ def visit_try_stmt(self, s: TryStmt) -> None: if s.finally_body: # First we check finally_body is type safe on all abnormal exit paths iter_errors = IterationDependentErrors() - with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors): self.accept(s.finally_body) if s.finally_body: @@ -4960,13 +4957,9 @@ def visit_try_stmt(self, s: TryStmt) -> None: # that follows the try statement.) assert iter_errors is not None if not self.binder.is_unreachable(): - with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher: + with IterationErrorWatcher(self.msg.errors, iter_errors): self.accept(s.finally_body) - - for error_info in watcher.yield_error_infos(): - self.msg.fail(*error_info[:2], code=error_info[2]) - for note_info in watcher.yield_note_infos(self.options): - self.msg.note(*note_info) + self.msg.iteration_dependent_errors(iter_errors) def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: """Type check a try statement, ignoring the finally block. diff --git a/mypy/errors.py b/mypy/errors.py index 5dd411c39e95..5c135146bcb7 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -15,6 +15,7 @@ from mypy.nodes import Context from mypy.options import Options from mypy.scope import Scope +from mypy.types import Type from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file from mypy.version import __version__ as mypy_version @@ -166,6 +167,10 @@ class ErrorWatcher: out by one of the ErrorWatcher instances. """ + # public attribute for the special treatment of `reveal_type` by + # `MessageBuilder.reveal_type`: + filter_revealed_type: bool + def __init__( self, errors: Errors, @@ -173,11 +178,13 @@ def __init__( filter_errors: bool | Callable[[str, ErrorInfo], bool] = False, save_filtered_errors: bool = False, filter_deprecated: bool = False, + filter_revealed_type: bool = False, ) -> None: self.errors = errors self._has_new_errors = False self._filter = filter_errors self._filter_deprecated = filter_deprecated + self.filter_revealed_type = filter_revealed_type self._filtered: list[ErrorInfo] | None = [] if save_filtered_errors else None def __enter__(self) -> Self: @@ -236,15 +243,41 @@ class IterationDependentErrors: # the error report occurs but really all unreachable lines. unreachable_lines: list[set[int]] - # One set of revealed types for each `reveal_type` statement. Each created set can - # grow during the iteration. Meaning of the tuple items: function_or_member, line, - # column, end_line, end_column: - revealed_types: dict[tuple[str | None, int, int, int, int], set[str]] + # One list of revealed types for each `reveal_type` statement. Each created list + # can grow during the iteration. Meaning of the tuple items: line, column, + # end_line, end_column: + revealed_types: dict[tuple[int, int, int | None, int | None], list[Type]] def __init__(self) -> None: self.uselessness_errors = [] self.unreachable_lines = [] - self.revealed_types = defaultdict(set) + self.revealed_types = defaultdict(list) + + def yield_uselessness_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]: + """Report only those `unreachable`, `redundant-expr`, and `redundant-casts` + errors that could not be ruled out in any iteration step.""" + + persistent_uselessness_errors = set() + for candidate in set(chain(*self.uselessness_errors)): + if all( + (candidate in errors) or (candidate[2] in lines) + for errors, lines in zip(self.uselessness_errors, self.unreachable_lines) + ): + persistent_uselessness_errors.add(candidate) + for error_info in persistent_uselessness_errors: + context = Context(line=error_info[2], column=error_info[3]) + context.end_line = error_info[4] + context.end_column = error_info[5] + yield error_info[1], context, error_info[0] + + def yield_revealed_type_infos(self) -> Iterator[tuple[list[Type], Context]]: + """Yield all types revealed in at least one iteration step.""" + + for note_info, types in self.revealed_types.items(): + context = Context(line=note_info[0], column=note_info[1]) + context.end_line = note_info[2] + context.end_column = note_info[3] + yield types, context class IterationErrorWatcher(ErrorWatcher): @@ -287,53 +320,8 @@ def on_error(self, file: str, info: ErrorInfo) -> bool: iter_errors.unreachable_lines[-1].update(range(info.line, info.end_line + 1)) return True - if info.code == codes.MISC and info.message.startswith("Revealed type is "): - key = info.function_or_member, info.line, info.column, info.end_line, info.end_column - types = info.message.split('"')[1] - if types.startswith("Union["): - iter_errors.revealed_types[key].update(types[6:-1].split(", ")) - else: - iter_errors.revealed_types[key].add(types) - return True - return super().on_error(file, info) - def yield_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]: - """Report only those `unreachable`, `redundant-expr`, and `redundant-casts` - errors that could not be ruled out in any iteration step.""" - - persistent_uselessness_errors = set() - iter_errors = self.iteration_dependent_errors - for candidate in set(chain(*iter_errors.uselessness_errors)): - if all( - (candidate in errors) or (candidate[2] in lines) - for errors, lines in zip( - iter_errors.uselessness_errors, iter_errors.unreachable_lines - ) - ): - persistent_uselessness_errors.add(candidate) - for error_info in persistent_uselessness_errors: - context = Context(line=error_info[2], column=error_info[3]) - context.end_line = error_info[4] - context.end_column = error_info[5] - yield error_info[1], context, error_info[0] - - def yield_note_infos(self, options: Options) -> Iterator[tuple[str, Context]]: - """Yield all types revealed in at least one iteration step.""" - - for note_info, types in self.iteration_dependent_errors.revealed_types.items(): - sorted_ = sorted(types, key=lambda typ: typ.lower()) - if len(types) == 1: - revealed = sorted_[0] - elif options.use_or_syntax(): - revealed = " | ".join(sorted_) - else: - revealed = f"Union[{', '.join(sorted_)}]" - context = Context(line=note_info[1], column=note_info[2]) - context.end_line = note_info[3] - context.end_column = note_info[4] - yield f'Revealed type is "{revealed}"', context - class Errors: """Container for compile errors. @@ -596,18 +584,19 @@ def _add_error_info(self, file: str, info: ErrorInfo) -> None: if info.code in (IMPORT, IMPORT_UNTYPED, IMPORT_NOT_FOUND): self.seen_import_error = True + def get_watchers(self) -> Iterator[ErrorWatcher]: + """Yield the `ErrorWatcher` stack from top to bottom.""" + i = len(self._watchers) + while i > 0: + i -= 1 + yield self._watchers[i] + def _filter_error(self, file: str, info: ErrorInfo) -> bool: """ process ErrorWatcher stack from top to bottom, stopping early if error needs to be filtered out """ - i = len(self._watchers) - while i > 0: - i -= 1 - w = self._watchers[i] - if w.on_error(file, info): - return True - return False + return any(w.on_error(file, info) for w in self.get_watchers()) def add_error_info(self, info: ErrorInfo) -> None: file, lines = info.origin diff --git a/mypy/messages.py b/mypy/messages.py index 01414f1c7f2b..1021a15e9145 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -23,7 +23,13 @@ from mypy import errorcodes as codes, message_registry from mypy.erasetype import erase_type from mypy.errorcodes import ErrorCode -from mypy.errors import ErrorInfo, Errors, ErrorWatcher +from mypy.errors import ( + ErrorInfo, + Errors, + ErrorWatcher, + IterationDependentErrors, + IterationErrorWatcher, +) from mypy.nodes import ( ARG_NAMED, ARG_NAMED_OPT, @@ -188,12 +194,14 @@ def filter_errors( filter_errors: bool | Callable[[str, ErrorInfo], bool] = True, save_filtered_errors: bool = False, filter_deprecated: bool = False, + filter_revealed_type: bool = False, ) -> ErrorWatcher: return ErrorWatcher( self.errors, filter_errors=filter_errors, save_filtered_errors=save_filtered_errors, filter_deprecated=filter_deprecated, + filter_revealed_type=filter_revealed_type, ) def add_errors(self, errors: list[ErrorInfo]) -> None: @@ -1735,6 +1743,24 @@ def invalid_signature_for_special_method( ) def reveal_type(self, typ: Type, context: Context) -> None: + + # Search for an error watcher that modifies the "normal" behaviour (we do not + # rely on the normal `ErrorWatcher` filtering approach because we might need to + # collect the original types for a later unionised response): + for watcher in self.errors.get_watchers(): + # The `reveal_type` statement should be ignored: + if watcher.filter_revealed_type: + return + # The `reveal_type` statement might be visited iteratively due to being + # placed in a loop or so. Hence, we collect the respective types of + # individual iterations so that we can report them all in one step later: + if isinstance(watcher, IterationErrorWatcher): + watcher.iteration_dependent_errors.revealed_types[ + (context.line, context.column, context.end_line, context.end_column) + ].append(typ) + return + + # Nothing special here; just create the note: visitor = TypeStrVisitor(options=self.options) self.note(f'Revealed type is "{typ.accept(visitor)}"', context) @@ -2478,6 +2504,12 @@ def match_statement_inexhaustive_match(self, typ: Type, context: Context) -> Non code=codes.EXHAUSTIVE_MATCH, ) + def iteration_dependent_errors(self, iter_errors: IterationDependentErrors) -> None: + for error_info in iter_errors.yield_uselessness_error_infos(): + self.fail(*error_info[:2], code=error_info[2]) + for types, context in iter_errors.yield_revealed_type_infos(): + self.reveal_type(mypy.typeops.make_simplified_union(types), context) + def quote_type_string(type_string: str) -> str: """Quotes a type representation for use in messages.""" diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 856d430a544c..b563eef0f8aa 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -343,7 +343,7 @@ for var2 in [g, h, i, j, k, l]: reveal_type(var2) # N: Revealed type is "Union[builtins.int, builtins.str]" for var3 in [m, n, o, p, q, r]: - reveal_type(var3) # N: Revealed type is "Union[Any, builtins.int]" + reveal_type(var3) # N: Revealed type is "Union[builtins.int, Any]" T = TypeVar("T", bound=Type[Foo]) @@ -1247,7 +1247,7 @@ class X(TypedDict): x: X for a in ("hourly", "daily"): - reveal_type(a) # N: Revealed type is "Union[Literal['daily']?, Literal['hourly']?]" + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" reveal_type(x[a]) # N: Revealed type is "builtins.int" reveal_type(a.upper()) # N: Revealed type is "builtins.str" c = a diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 36a148fc47df..4b31835da743 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2346,7 +2346,7 @@ def f() -> bool: ... y = None while f(): - reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(y) # N: Revealed type is "Union[None, builtins.int]" y = 1 reveal_type(y) # N: Revealed type is "Union[builtins.int, None]" diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 924e66584669..3523772611aa 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -628,7 +628,7 @@ def f1() -> None: def f2() -> None: x = None while int(): - reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" + reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" if int(): x = "" reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" @@ -923,7 +923,7 @@ class X(TypedDict): x: X for a in ("hourly", "daily"): - reveal_type(a) # N: Revealed type is "Union[Literal['daily']?, Literal['hourly']?]" + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" reveal_type(x[a]) # N: Revealed type is "builtins.int" reveal_type(a.upper()) # N: Revealed type is "builtins.str" c = a diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 0f69d0a56f47..41e90c3f8506 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -989,7 +989,7 @@ from typing_extensions import Unpack def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[float, ...]], bool]]) -> None: for x in xs: - reveal_type(x) # N: Revealed type is "Union[builtins.float, builtins.int]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] [case testFixedUnpackItemInInstanceArguments] diff --git a/test-data/unit/check-union-error-syntax.test b/test-data/unit/check-union-error-syntax.test index d41281b774e1..e938598aaefe 100644 --- a/test-data/unit/check-union-error-syntax.test +++ b/test-data/unit/check-union-error-syntax.test @@ -62,17 +62,18 @@ x = 3 # E: Incompatible types in assignment (expression has type "Literal[3]", v try: x = 1 x = "" + x = {1: ""} finally: - reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, builtins.dict[builtins.int, builtins.str]]" [builtins fixtures/isinstancelist.pyi] [case testOrSyntaxRecombined] # flags: --python-version 3.10 --no-force-union-syntax --allow-redefinition-new --local-partial-types # The following revealed type is recombined because the finally body is visited twice. -# ToDo: Improve this recombination logic, especially (but not only) for the "or syntax". try: x = 1 x = "" + x = {1: ""} finally: - reveal_type(x) # N: Revealed type is "builtins.int | builtins.str | builtins.str" + reveal_type(x) # N: Revealed type is "builtins.int | builtins.str | builtins.dict[builtins.int, builtins.str]" [builtins fixtures/isinstancelist.pyi] From 7d133961a7e759aab84223bf8038b9489daaa93c Mon Sep 17 00:00:00 2001 From: esarp <11684270+esarp@users.noreply.github.com> Date: Mon, 14 Jul 2025 10:34:40 -0500 Subject: [PATCH 08/15] Initial changelog for 1.17 release (#19427) --- CHANGELOG.md | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1470b7d50c3..e4f148fe6382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Next Release +## Mypy 1.17 (Unreleased) + +We’ve just uploaded mypy 1.17 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features and bug fixes. +You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + ### Remove Support for targeting Python 3.8 Mypy now requires `--python-version 3.9` or greater. Support for only Python 3.8 is @@ -29,6 +39,119 @@ Mypy only supports Python 3.9+. The \--force-uppercase-builtins flag is now depr Contributed by Marc Mueller (PR [19176](https://github.com/python/mypy/pull/19176)) +### Mypyc Fixes and Improvements + +* Fix exception swallowing in async try/finally blocks with await (Chainfire, PR [19353](https://github.com/python/mypy/pull/19353)) +* Fix AttributeError in async try/finally with mixed return paths (Chainfire, PR [19361](https://github.com/python/mypy/pull/19361)) +* Derive .c file name from full module name if using multi_file (Jukka Lehtosalo, PR [19278](https://github.com/python/mypy/pull/19278)) +* Support overriding the group name used in output files (Jukka Lehtosalo, PR [19272](https://github.com/python/mypy/pull/19272)) +* Make generated generator helper method internal (Jukka Lehtosalo, PR [19268](https://github.com/python/mypy/pull/19268)) +* Add note about using non-native class to subclass built-in types (Jukka Lehtosalo, PR [19236](https://github.com/python/mypy/pull/19236)) +* Make some generated classes implicitly final (Jukka Lehtosalo, PR [19235](https://github.com/python/mypy/pull/19235)) +* Free coroutine after await encounters StopIteration (Jukka Lehtosalo, PR [19231](https://github.com/python/mypy/pull/19231)) +* Use non-tagged integer for generator label (Jukka Lehtosalo, PR [19218](https://github.com/python/mypy/pull/19218)) +* Merge generator and environment classes in simple cases (Jukka Lehtosalo, PR [19207](https://github.com/python/mypy/pull/19207)) +* Don't simplify module prefixes if using separate compilation (Jukka Lehtosalo, PR [19206](https://github.com/python/mypy/pull/19206)) +* Test function nesting with async functions (Jukka Lehtosalo, PR [19203](https://github.com/python/mypy/pull/19203)) +* Enable partial, unsafe support for free-threading (Jukka Lehtosalo, PR [19167](https://github.com/python/mypy/pull/19167)) +* Add comment about incref/decref and free-threaded builds (Jukka Lehtosalo, PR [19155](https://github.com/python/mypy/pull/19155)) +* Refactor extension module C generation and generated C (Jukka Lehtosalo, PR [19126](https://github.com/python/mypy/pull/19126)) +* Fix incref/decref on free-threaded builds (Jukka Lehtosalo, PR [19127](https://github.com/python/mypy/pull/19127)) +* Remove last unreachable block from mypyc code (Stanislav Terliakov, PR [19086](https://github.com/python/mypy/pull/19086)) + +### Stubgen Improvements + +* stubgen: add test case for handling `Incomplete` return types (Alexey Makridenko, PR [19253](https://github.com/python/mypy/pull/19253)) +* stubgen: add import for `types` in `__exit__` method signature (Alexey Makridenko, PR [19120](https://github.com/python/mypy/pull/19120)) +* stubgenc: add support for including class and property docstrings (Chad Dombrova, PR [17964](https://github.com/python/mypy/pull/17964)) +* stubgen: Don't generate `Incomplete | None = None` argument annotation (Sebastian Rittau, PR [19097](https://github.com/python/mypy/pull/19097)) +* Support several more constructs in stubgen's AliasPrinter (Stanislav Terliakov, PR [18888](https://github.com/python/mypy/pull/18888)) + +### Stubtest Improvements + +* Syntax error messages capitalization (Charulata, PR [19114](https://github.com/python/mypy/pull/19114)) + +### Miscellaneous Fixes and Improvements + +* Combine the revealed types of multiple iteration steps in a more robust manner (Christoph Tyralla, PR [19324](https://github.com/python/mypy/pull/19324)) +* Improve the handling of "iteration dependent" errors and notes in finally clauses (Christoph Tyralla, PR [19270](https://github.com/python/mypy/pull/19270)) +* Lessen dmypy suggest path limitations for Windows machines (CoolCat467, PR [19337](https://github.com/python/mypy/pull/19337)) +* Type ignore comments erroneously marked as unused by dmypy (Charlie Denton, PR [15043](https://github.com/python/mypy/pull/15043)) +* Handle corner case: protocol vs classvar vs descriptor (Ivan Levkivskyi, PR [19277](https://github.com/python/mypy/pull/19277)) +* Fix `exhaustive-match` error code in title (johnthagen, PR [19276](https://github.com/python/mypy/pull/19276)) +* Fix couple inconsistencies in protocols vs TypeType (Ivan Levkivskyi, PR [19267](https://github.com/python/mypy/pull/19267)) +* Fix missing error context for unpacking assignment involving star expression (Brian Schubert, PR [19258](https://github.com/python/mypy/pull/19258)) +* Fix and simplify error de-duplication (Ivan Levkivskyi, PR [19247](https://github.com/python/mypy/pull/19247)) +* Add regression test for narrowing union of mixins (Shantanu, PR [19266](https://github.com/python/mypy/pull/19266)) +* Disallow `ClassVar` in type aliases (Brian Schubert, PR [19263](https://github.com/python/mypy/pull/19263)) +* Refactor/unify access to static attributes (Ivan Levkivskyi, PR [19254](https://github.com/python/mypy/pull/19254)) +* Clean-up and move operator access to checkmember.py (Ivan Levkivskyi, PR [19250](https://github.com/python/mypy/pull/19250)) +* Add script that prints compiled files when self compiling (Jukka Lehtosalo, PR [19260](https://github.com/python/mypy/pull/19260)) +* Fix help message url for "None and Optional handling" section (Guy Wilson, PR [19252](https://github.com/python/mypy/pull/19252)) +* Display FQN for imported base classes in errors about incompatible overrides (Mikhail Golubev, PR [19115](https://github.com/python/mypy/pull/19115)) +* Fix a minor merge conflict caused by #19118 (Christoph Tyralla, PR [19246](https://github.com/python/mypy/pull/19246)) +* Avoid false `unreachable`, `redundant-expr`, and `redundant-casts` warnings in loops more robustly and efficiently, and avoid multiple `revealed type` notes for the same line (Christoph Tyralla, PR [19118](https://github.com/python/mypy/pull/19118)) +* Fix type extraction from `isinstance` checks (Stanislav Terliakov, PR [19223](https://github.com/python/mypy/pull/19223)) +* Erase stray typevars in functools.partial generic (Stanislav Terliakov, PR [18954](https://github.com/python/mypy/pull/18954)) +* Make infer_condition_value recognize the whole truth table (Stanislav Terliakov, PR [18944](https://github.com/python/mypy/pull/18944)) +* Support type aliases, `NamedTuple` and `TypedDict` in constrained TypeVar defaults (Stanislav Terliakov, PR [18884](https://github.com/python/mypy/pull/18884)) +* Move dataclass kw_only fields to the end of the signature (Stanislav Terliakov, PR [19018](https://github.com/python/mypy/pull/19018)) +* Deprecated --force-uppercase-builtins flag (Marc Mueller, PR [19176](https://github.com/python/mypy/pull/19176)) +* Provide a better fallback value for the python_version option (Marc Mueller, PR [19162](https://github.com/python/mypy/pull/19162)) +* Avoid spurious non-overlapping eq error with metaclass with `__eq__` (Michael J. Sullivan, PR [19220](https://github.com/python/mypy/pull/19220)) +* Remove --show-speed-regression in primer (Shantanu, PR [19226](https://github.com/python/mypy/pull/19226)) +* Add flag to raise error if match statement does not match exaustively (Donal Burns, PR [19144](https://github.com/python/mypy/pull/19144)) +* Narrow type variable bounds in binder (Ivan Levkivskyi, PR [19183](https://github.com/python/mypy/pull/19183)) +* Add regression test for dataclass typeguard (Shantanu, PR [19214](https://github.com/python/mypy/pull/19214)) +* Add classifier for Python 3.14 (Marc Mueller, PR [19199](https://github.com/python/mypy/pull/19199)) +* Further cleanup after dropping Python 3.8 (Marc Mueller, PR [19197](https://github.com/python/mypy/pull/19197)) +* Fix nondeterministic type checking by making join with explicit Protocol and type promotion commute (Shantanu, PR [18402](https://github.com/python/mypy/pull/18402)) +* Infer constraints eagerly if actual is Any (Ivan Levkivskyi, PR [19190](https://github.com/python/mypy/pull/19190)) +* Include walrus assignments in conditional inference (Stanislav Terliakov, PR [19038](https://github.com/python/mypy/pull/19038)) +* Use PEP 604 syntax for TypeStrVisitor (Marc Mueller, PR [19179](https://github.com/python/mypy/pull/19179)) +* Use checkmember.py to check protocol subtyping (Ivan Levkivskyi, PR [18943](https://github.com/python/mypy/pull/18943)) +* Update test requirements (Marc Mueller, PR [19163](https://github.com/python/mypy/pull/19163)) +* Use more lower case builtins in error messages (Marc Mueller, PR [19177](https://github.com/python/mypy/pull/19177)) +* Remove force_uppercase_builtins default from test helpers (Marc Mueller, PR [19173](https://github.com/python/mypy/pull/19173)) +* Start testing Python 3.14 (Marc Mueller, PR [19164](https://github.com/python/mypy/pull/19164)) +* Fix example to use correct method of Stack (Łukasz Kwieciński, PR [19123](https://github.com/python/mypy/pull/19123)) +* Fix nondeterministic type checking caused by nonassociative of None joins (Shantanu, PR [19158](https://github.com/python/mypy/pull/19158)) +* Drop support for --python-version 3.8 (Marc Mueller, PR [19157](https://github.com/python/mypy/pull/19157)) +* Fix nondeterministic type checking caused by nonassociativity of joins (Shantanu, PR [19147](https://github.com/python/mypy/pull/19147)) +* Fix nondeterministic type checking by making join between TypeType and TypeVar commute (Shantanu, PR [19149](https://github.com/python/mypy/pull/19149)) +* Forbid `.pop` of `Readonly` `NotRequired` TypedDict items (Stanislav Terliakov, PR [19133](https://github.com/python/mypy/pull/19133)) +* Emit a friendlier warning on invalid exclude regex, instead of a stacktrace (wyattscarpenter, PR [19102](https://github.com/python/mypy/pull/19102)) +* Update dmypy/client.py: Enable ANSI color codes for windows cmd (wyattscarpenter, PR [19088](https://github.com/python/mypy/pull/19088)) +* Extend special case for context-based typevar inference to typevar unions in return position (Stanislav Terliakov, PR [18976](https://github.com/python/mypy/pull/18976)) + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +* Alexey Makridenko +* Brian Schubert +* Chad Dombrova +* Chainfire +* Charlie Denton +* Charulata +* Christoph Tyralla +* CoolCat467 +* Donal Burns +* Guy Wilson +* Ivan Levkivskyi +* johnthagen +* Jukka Lehtosalo +* Łukasz Kwieciński +* Marc Mueller +* Michael J. Sullivan +* Mikhail Golubev +* Sebastian Rittau +* Shantanu +* Stanislav Terliakov +* wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.16 We’ve just uploaded mypy 1.16 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From 3901aa2f9523ce55e08d94c1716028d840398753 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 14 Jul 2025 16:51:20 +0100 Subject: [PATCH 09/15] Updates to 1.17 changelog (#19436) Add a few sections and do some editing. --- CHANGELOG.md | 168 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 107 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4f148fe6382..a74fb46aba6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Next Release -## Mypy 1.17 (Unreleased) +## Mypy 1.17 We’ve just uploaded mypy 1.17 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features and bug fixes. @@ -12,11 +12,60 @@ You can install it as follows: You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -### Remove Support for targeting Python 3.8 +### Optionally Check That Match Is Exhaustive -Mypy now requires `--python-version 3.9` or greater. Support for only Python 3.8 is -fully removed now. Given an unsupported version, mypy will default to the oldest -supported one, currently 3.9. +Mypy can now optionally generate an error if a match statement does not +match exhaustively, without having to use `assert_never(...)`. Enable +this by using `--enable-error-code exhaustive-match`. + +Example: + +```python +# mypy: enable-error-code=exhaustive-match + +import enum + +class Color(enum.Enum): + RED = 1 + BLUE = 2 + +def show_color(val: Color) -> None: + # error: Unhandled case for values of type "Literal[Color.BLUE]" + match val: + case Color.RED: + print("red") +``` + +This feature was contributed by Donal Burns (PR [19144](https://github.com/python/mypy/pull/19144)). + +### Further Improvements to Attribute Resolution + +This release includes additional improvements to how attribute types +and kinds are resolved. These fix many bugs and overall improve consistency. + +* Handle corner case: protocol/class variable/descriptor (Ivan Levkivskyi, PR [19277](https://github.com/python/mypy/pull/19277)) +* Fix a few inconsistencies in protocol/type object interactions (Ivan Levkivskyi, PR [19267](https://github.com/python/mypy/pull/19267)) +* Refactor/unify access to static attributes (Ivan Levkivskyi, PR [19254](https://github.com/python/mypy/pull/19254)) +* Remove inconsistencies in operator handling (Ivan Levkivskyi, PR [19250](https://github.com/python/mypy/pull/19250)) +* Make protocol subtyping more consistent (Ivan Levkivskyi, PR [18943](https://github.com/python/mypy/pull/18943)) + +### Fixes to Nondeterministic Type Checking + +Previous mypy versions could infer different types for certain expressions +across different runs (typically depending on which order certain types +were processed, and this order was nondeterministic). This release includes +fixes to several such issues. + +* Fix nondeterministic type checking by making join with explicit Protocol and type promotion commute (Shantanu, PR [18402](https://github.com/python/mypy/pull/18402)) +* Fix nondeterministic type checking caused by nonassociative of None joins (Shantanu, PR [19158](https://github.com/python/mypy/pull/19158)) +* Fix nondeterministic type checking caused by nonassociativity of joins (Shantanu, PR [19147](https://github.com/python/mypy/pull/19147)) +* Fix nondeterministic type checking by making join between `type` and TypeVar commute (Shantanu, PR [19149](https://github.com/python/mypy/pull/19149)) + +### Remove Support for Targeting Python 3.8 + +Mypy now requires `--python-version 3.9` or greater. Support for targeting Python 3.8 is +fully removed now. Since 3.8 is an unsupported version, mypy will default to the oldest +supported version (currently 3.9) if you still try to target 3.8. This change is necessary because typeshed stopped supporting Python 3.8 after it reached its End of Life in October 2024. @@ -27,102 +76,99 @@ Contributed by Marc Mueller ### Initial Support for Python 3.14 Mypy is now tested on 3.14 and mypyc works with 3.14.0b3 and later. -Mypyc compiled wheels of mypy itself will be available for new versions after 3.14.0rc1 is released. +Binary wheels compiled with mypyc for mypy itself will be available for 3.14 +some time after 3.14.0rc1 has been released. -Note that not all new features might be supported just yet. +Note that not all features are supported just yet. Contributed by Marc Mueller (PR [19164](https://github.com/python/mypy/pull/19164)) -### Deprecated Flag: \--force-uppercase-builtins +### Deprecated Flag: `--force-uppercase-builtins` -Mypy only supports Python 3.9+. The \--force-uppercase-builtins flag is now deprecated and a no-op. It will be removed in a future version. +Mypy only supports Python 3.9+. The `--force-uppercase-builtins` flag is now +deprecated as unnecessary, and a no-op. It will be removed in a future version. Contributed by Marc Mueller (PR [19176](https://github.com/python/mypy/pull/19176)) -### Mypyc Fixes and Improvements +### Mypyc: Improvements to Generators and Async Functions + +This release includes both performance improvements and bug fixes related +to generators and async functions (these share many implementation details). * Fix exception swallowing in async try/finally blocks with await (Chainfire, PR [19353](https://github.com/python/mypy/pull/19353)) * Fix AttributeError in async try/finally with mixed return paths (Chainfire, PR [19361](https://github.com/python/mypy/pull/19361)) -* Derive .c file name from full module name if using multi_file (Jukka Lehtosalo, PR [19278](https://github.com/python/mypy/pull/19278)) -* Support overriding the group name used in output files (Jukka Lehtosalo, PR [19272](https://github.com/python/mypy/pull/19272)) * Make generated generator helper method internal (Jukka Lehtosalo, PR [19268](https://github.com/python/mypy/pull/19268)) -* Add note about using non-native class to subclass built-in types (Jukka Lehtosalo, PR [19236](https://github.com/python/mypy/pull/19236)) -* Make some generated classes implicitly final (Jukka Lehtosalo, PR [19235](https://github.com/python/mypy/pull/19235)) * Free coroutine after await encounters StopIteration (Jukka Lehtosalo, PR [19231](https://github.com/python/mypy/pull/19231)) * Use non-tagged integer for generator label (Jukka Lehtosalo, PR [19218](https://github.com/python/mypy/pull/19218)) * Merge generator and environment classes in simple cases (Jukka Lehtosalo, PR [19207](https://github.com/python/mypy/pull/19207)) -* Don't simplify module prefixes if using separate compilation (Jukka Lehtosalo, PR [19206](https://github.com/python/mypy/pull/19206)) -* Test function nesting with async functions (Jukka Lehtosalo, PR [19203](https://github.com/python/mypy/pull/19203)) + +### Mypyc: Partial, Unsafe Support for Free Threading + +Mypyc has minimal, quite memory-unsafe support for the free threaded +builds of 3.14. It is also only lightly tested. Bug reports and experience +reports are welcome! + +Here are some of the major limitations: +* Free threading only works when compiling a single module at a time. +* If there is concurrent access to an object while another thread is mutating the same + object, it's possible to encounter segfaults and memory corruption. +* There are no efficient native primitives for thread synthronization, though the + regular `threading` module can be used. +* Some workloads don't scale well to multiple threads for no clear reason. + +Related PRs: + * Enable partial, unsafe support for free-threading (Jukka Lehtosalo, PR [19167](https://github.com/python/mypy/pull/19167)) -* Add comment about incref/decref and free-threaded builds (Jukka Lehtosalo, PR [19155](https://github.com/python/mypy/pull/19155)) -* Refactor extension module C generation and generated C (Jukka Lehtosalo, PR [19126](https://github.com/python/mypy/pull/19126)) * Fix incref/decref on free-threaded builds (Jukka Lehtosalo, PR [19127](https://github.com/python/mypy/pull/19127)) -* Remove last unreachable block from mypyc code (Stanislav Terliakov, PR [19086](https://github.com/python/mypy/pull/19086)) -### Stubgen Improvements +### Other Mypyc Fixes and Improvements -* stubgen: add test case for handling `Incomplete` return types (Alexey Makridenko, PR [19253](https://github.com/python/mypy/pull/19253)) -* stubgen: add import for `types` in `__exit__` method signature (Alexey Makridenko, PR [19120](https://github.com/python/mypy/pull/19120)) -* stubgenc: add support for including class and property docstrings (Chad Dombrova, PR [17964](https://github.com/python/mypy/pull/17964)) -* stubgen: Don't generate `Incomplete | None = None` argument annotation (Sebastian Rittau, PR [19097](https://github.com/python/mypy/pull/19097)) -* Support several more constructs in stubgen's AliasPrinter (Stanislav Terliakov, PR [18888](https://github.com/python/mypy/pull/18888)) +* Derive .c file name from full module name if using multi_file (Jukka Lehtosalo, PR [19278](https://github.com/python/mypy/pull/19278)) +* Support overriding the group name used in output files (Jukka Lehtosalo, PR [19272](https://github.com/python/mypy/pull/19272)) +* Add note about using non-native class to subclass built-in types (Jukka Lehtosalo, PR [19236](https://github.com/python/mypy/pull/19236)) +* Make some generated classes implicitly final (Jukka Lehtosalo, PR [19235](https://github.com/python/mypy/pull/19235)) +* Don't simplify module prefixes if using separate compilation (Jukka Lehtosalo, PR [19206](https://github.com/python/mypy/pull/19206)) -### Stubtest Improvements +### Stubgen Improvements -* Syntax error messages capitalization (Charulata, PR [19114](https://github.com/python/mypy/pull/19114)) +* Add import for `types` in `__exit__` method signature (Alexey Makridenko, PR [19120](https://github.com/python/mypy/pull/19120)) +* Add support for including class and property docstrings (Chad Dombrova, PR [17964](https://github.com/python/mypy/pull/17964)) +* Don't generate `Incomplete | None = None` argument annotation (Sebastian Rittau, PR [19097](https://github.com/python/mypy/pull/19097)) +* Support several more constructs in stubgen's alias printer (Stanislav Terliakov, PR [18888](https://github.com/python/mypy/pull/18888)) ### Miscellaneous Fixes and Improvements * Combine the revealed types of multiple iteration steps in a more robust manner (Christoph Tyralla, PR [19324](https://github.com/python/mypy/pull/19324)) * Improve the handling of "iteration dependent" errors and notes in finally clauses (Christoph Tyralla, PR [19270](https://github.com/python/mypy/pull/19270)) * Lessen dmypy suggest path limitations for Windows machines (CoolCat467, PR [19337](https://github.com/python/mypy/pull/19337)) -* Type ignore comments erroneously marked as unused by dmypy (Charlie Denton, PR [15043](https://github.com/python/mypy/pull/15043)) -* Handle corner case: protocol vs classvar vs descriptor (Ivan Levkivskyi, PR [19277](https://github.com/python/mypy/pull/19277)) -* Fix `exhaustive-match` error code in title (johnthagen, PR [19276](https://github.com/python/mypy/pull/19276)) -* Fix couple inconsistencies in protocols vs TypeType (Ivan Levkivskyi, PR [19267](https://github.com/python/mypy/pull/19267)) +* Fix type ignore comments erroneously marked as unused by dmypy (Charlie Denton, PR [15043](https://github.com/python/mypy/pull/15043)) +* Fix misspelled `exhaustive-match` error code (johnthagen, PR [19276](https://github.com/python/mypy/pull/19276)) * Fix missing error context for unpacking assignment involving star expression (Brian Schubert, PR [19258](https://github.com/python/mypy/pull/19258)) * Fix and simplify error de-duplication (Ivan Levkivskyi, PR [19247](https://github.com/python/mypy/pull/19247)) -* Add regression test for narrowing union of mixins (Shantanu, PR [19266](https://github.com/python/mypy/pull/19266)) * Disallow `ClassVar` in type aliases (Brian Schubert, PR [19263](https://github.com/python/mypy/pull/19263)) -* Refactor/unify access to static attributes (Ivan Levkivskyi, PR [19254](https://github.com/python/mypy/pull/19254)) -* Clean-up and move operator access to checkmember.py (Ivan Levkivskyi, PR [19250](https://github.com/python/mypy/pull/19250)) -* Add script that prints compiled files when self compiling (Jukka Lehtosalo, PR [19260](https://github.com/python/mypy/pull/19260)) +* Add script that prints list of compiled files when compiling mypy (Jukka Lehtosalo, PR [19260](https://github.com/python/mypy/pull/19260)) * Fix help message url for "None and Optional handling" section (Guy Wilson, PR [19252](https://github.com/python/mypy/pull/19252)) -* Display FQN for imported base classes in errors about incompatible overrides (Mikhail Golubev, PR [19115](https://github.com/python/mypy/pull/19115)) -* Fix a minor merge conflict caused by #19118 (Christoph Tyralla, PR [19246](https://github.com/python/mypy/pull/19246)) +* Display fully qualified name of imported base classes in errors about incompatible overrides (Mikhail Golubev, PR [19115](https://github.com/python/mypy/pull/19115)) * Avoid false `unreachable`, `redundant-expr`, and `redundant-casts` warnings in loops more robustly and efficiently, and avoid multiple `revealed type` notes for the same line (Christoph Tyralla, PR [19118](https://github.com/python/mypy/pull/19118)) * Fix type extraction from `isinstance` checks (Stanislav Terliakov, PR [19223](https://github.com/python/mypy/pull/19223)) -* Erase stray typevars in functools.partial generic (Stanislav Terliakov, PR [18954](https://github.com/python/mypy/pull/18954)) -* Make infer_condition_value recognize the whole truth table (Stanislav Terliakov, PR [18944](https://github.com/python/mypy/pull/18944)) +* Erase stray type variables in `functools.partial` (Stanislav Terliakov, PR [18954](https://github.com/python/mypy/pull/18954)) +* Make inferring condition value recognize the whole truth table (Stanislav Terliakov, PR [18944](https://github.com/python/mypy/pull/18944)) * Support type aliases, `NamedTuple` and `TypedDict` in constrained TypeVar defaults (Stanislav Terliakov, PR [18884](https://github.com/python/mypy/pull/18884)) -* Move dataclass kw_only fields to the end of the signature (Stanislav Terliakov, PR [19018](https://github.com/python/mypy/pull/19018)) -* Deprecated --force-uppercase-builtins flag (Marc Mueller, PR [19176](https://github.com/python/mypy/pull/19176)) -* Provide a better fallback value for the python_version option (Marc Mueller, PR [19162](https://github.com/python/mypy/pull/19162)) -* Avoid spurious non-overlapping eq error with metaclass with `__eq__` (Michael J. Sullivan, PR [19220](https://github.com/python/mypy/pull/19220)) -* Remove --show-speed-regression in primer (Shantanu, PR [19226](https://github.com/python/mypy/pull/19226)) -* Add flag to raise error if match statement does not match exaustively (Donal Burns, PR [19144](https://github.com/python/mypy/pull/19144)) -* Narrow type variable bounds in binder (Ivan Levkivskyi, PR [19183](https://github.com/python/mypy/pull/19183)) -* Add regression test for dataclass typeguard (Shantanu, PR [19214](https://github.com/python/mypy/pull/19214)) +* Move dataclass `kw_only` fields to the end of the signature (Stanislav Terliakov, PR [19018](https://github.com/python/mypy/pull/19018)) +* Provide a better fallback value for the `python_version` option (Marc Mueller, PR [19162](https://github.com/python/mypy/pull/19162)) +* Avoid spurious non-overlapping equality error with metaclass with `__eq__` (Michael J. Sullivan, PR [19220](https://github.com/python/mypy/pull/19220)) +* Narrow type variable bounds (Ivan Levkivskyi, PR [19183](https://github.com/python/mypy/pull/19183)) * Add classifier for Python 3.14 (Marc Mueller, PR [19199](https://github.com/python/mypy/pull/19199)) -* Further cleanup after dropping Python 3.8 (Marc Mueller, PR [19197](https://github.com/python/mypy/pull/19197)) -* Fix nondeterministic type checking by making join with explicit Protocol and type promotion commute (Shantanu, PR [18402](https://github.com/python/mypy/pull/18402)) +* Capitalize syntax error messages (Charulata, PR [19114](https://github.com/python/mypy/pull/19114)) * Infer constraints eagerly if actual is Any (Ivan Levkivskyi, PR [19190](https://github.com/python/mypy/pull/19190)) * Include walrus assignments in conditional inference (Stanislav Terliakov, PR [19038](https://github.com/python/mypy/pull/19038)) -* Use PEP 604 syntax for TypeStrVisitor (Marc Mueller, PR [19179](https://github.com/python/mypy/pull/19179)) -* Use checkmember.py to check protocol subtyping (Ivan Levkivskyi, PR [18943](https://github.com/python/mypy/pull/18943)) -* Update test requirements (Marc Mueller, PR [19163](https://github.com/python/mypy/pull/19163)) -* Use more lower case builtins in error messages (Marc Mueller, PR [19177](https://github.com/python/mypy/pull/19177)) -* Remove force_uppercase_builtins default from test helpers (Marc Mueller, PR [19173](https://github.com/python/mypy/pull/19173)) -* Start testing Python 3.14 (Marc Mueller, PR [19164](https://github.com/python/mypy/pull/19164)) +* Use PEP 604 syntax when converting types to strings (Marc Mueller, PR [19179](https://github.com/python/mypy/pull/19179)) +* Use more lower-case builtin types in error messages (Marc Mueller, PR [19177](https://github.com/python/mypy/pull/19177)) * Fix example to use correct method of Stack (Łukasz Kwieciński, PR [19123](https://github.com/python/mypy/pull/19123)) -* Fix nondeterministic type checking caused by nonassociative of None joins (Shantanu, PR [19158](https://github.com/python/mypy/pull/19158)) -* Drop support for --python-version 3.8 (Marc Mueller, PR [19157](https://github.com/python/mypy/pull/19157)) -* Fix nondeterministic type checking caused by nonassociativity of joins (Shantanu, PR [19147](https://github.com/python/mypy/pull/19147)) -* Fix nondeterministic type checking by making join between TypeType and TypeVar commute (Shantanu, PR [19149](https://github.com/python/mypy/pull/19149)) * Forbid `.pop` of `Readonly` `NotRequired` TypedDict items (Stanislav Terliakov, PR [19133](https://github.com/python/mypy/pull/19133)) * Emit a friendlier warning on invalid exclude regex, instead of a stacktrace (wyattscarpenter, PR [19102](https://github.com/python/mypy/pull/19102)) -* Update dmypy/client.py: Enable ANSI color codes for windows cmd (wyattscarpenter, PR [19088](https://github.com/python/mypy/pull/19088)) -* Extend special case for context-based typevar inference to typevar unions in return position (Stanislav Terliakov, PR [18976](https://github.com/python/mypy/pull/18976)) +* Enable ANSI color codes for dmypy client in Windows (wyattscarpenter, PR [19088](https://github.com/python/mypy/pull/19088)) +* Extend special case for context-based type variable inference to unions in return position (Stanislav Terliakov, PR [18976](https://github.com/python/mypy/pull/18976)) ### Acknowledgements From 0260991f6b055110c3df36bd5539d4f4489bf153 Mon Sep 17 00:00:00 2001 From: Ethan Sarp <11684270+esarp@users.noreply.github.com> Date: Mon, 14 Jul 2025 11:45:00 -0500 Subject: [PATCH 10/15] Update version string --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 21d23758c6dc..60b64f938241 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.17.0+dev" +__version__ = "1.17.0" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From e44d14f8e52a8890d08726ee753c8754edefd649 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 26 Jul 2025 18:56:37 -0700 Subject: [PATCH 11/15] Bump version to 1.17.1+dev --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 60b64f938241..a305b237a771 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.17.0" +__version__ = "1.17.1+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From 88fdeaae0abc92d605fc475fd153b4ad5b239310 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 16 Jul 2025 03:10:02 +0200 Subject: [PATCH 12/15] Prevent a crash when InitVar is redefined with a method in a subclass (#19453) Fixes #19443. This case is too niche (and should be trivially avoidable), so just not crashing should be good enough. The value is indeed redefined, and trying to massage the plugin to move the `X-redefinition` back to `X` in names is not worth the effort IMO. --- mypy/checker.py | 9 ++++++++- test-data/unit/check-dataclasses.test | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 217a4a885dd8..d2da9c51f80b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2169,7 +2169,14 @@ def check_method_override_for_base_with_name( else: override_class_or_static = defn.func.is_class or defn.func.is_static typ, _ = self.node_type_from_base(defn.name, defn.info, defn) - assert typ is not None + if typ is None: + # This may only happen if we're checking `x-redefinition` member + # and `x` itself is for some reason gone. Normally the node should + # be reachable from the containing class by its name. + # The redefinition is never removed, use this as a sanity check to verify + # the reasoning above. + assert f"{defn.name}-redefinition" in defn.info.names + return False original_node = base_attr.node # `original_type` can be partial if (e.g.) it is originally an diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 30d8497c9cd2..90be3707eed4 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2666,3 +2666,19 @@ class PersonBad(TypedDict): class JobBad: person: PersonBad = field(default_factory=PersonBad) # E: Argument "default_factory" to "field" has incompatible type "type[PersonBad]"; expected "Callable[[], PersonBad]" [builtins fixtures/dict.pyi] + +[case testDataclassInitVarRedefinitionNoCrash] +# https://github.com/python/mypy/issues/19443 +from dataclasses import InitVar, dataclass + +class ClassA: + def value(self) -> int: + return 0 + +@dataclass +class ClassB(ClassA): + value: InitVar[int] + + def value(self) -> int: # E: Name "value" already defined on line 10 + return 0 +[builtins fixtures/dict.pyi] From 5f4428f0286df58169d2f34f4f86561ad617538b Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 18 Jul 2025 16:41:41 +0200 Subject: [PATCH 13/15] Fix "ignored exception in `hasattr`" in dmypy (#19428) Fixes #19425. That property has no setter so it should safe to exclude. It can raise in inconsistent state that arises when it is not copied last? --- mypy/server/astmerge.py | 15 ++++++++++----- pyproject.toml | 8 ++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 8cd574628bb8..33e2d2b799cb 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -345,13 +345,11 @@ def visit_type_alias(self, node: TypeAlias) -> None: def fixup(self, node: SN) -> SN: if node in self.replacements: new = self.replacements[node] - skip_slots: tuple[str, ...] = () if isinstance(node, TypeInfo) and isinstance(new, TypeInfo): # Special case: special_alias is not exposed in symbol tables, but may appear # in external types (e.g. named tuples), so we need to update it manually. - skip_slots = ("special_alias",) replace_object_state(new.special_alias, node.special_alias) - replace_object_state(new, node, skip_slots=skip_slots) + replace_object_state(new, node, skip_slots=_get_ignored_slots(new)) return cast(SN, new) return node @@ -556,9 +554,16 @@ def replace_nodes_in_symbol_table( if node.node in replacements: new = replacements[node.node] old = node.node - # Needed for TypeInfo, see comment in fixup() above. - replace_object_state(new, old, skip_slots=("special_alias",)) + replace_object_state(new, old, skip_slots=_get_ignored_slots(new)) node.node = new if isinstance(node.node, (Var, TypeAlias)): # Handle them here just in case these aren't exposed through the AST. node.node.accept(NodeReplaceVisitor(replacements)) + + +def _get_ignored_slots(node: SymbolNode) -> tuple[str, ...]: + if isinstance(node, OverloadedFuncDef): + return ("setter",) + if isinstance(node, TypeInfo): + return ("special_alias",) + return () diff --git a/pyproject.toml b/pyproject.toml index 1870e0931407..032bfcb609e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,6 +222,14 @@ addopts = "-nauto --strict-markers --strict-config" # treat xpasses as test failures so they get converted to regular tests as soon as possible xfail_strict = true +# Force warnings as errors +filterwarnings = [ + "error", + # Some testcases may contain code that emits SyntaxWarnings, and they are not yet + # handled consistently in 3.14 (PEP 765) + "default::SyntaxWarning", +] + [tool.coverage.run] branch = true source = ["mypy"] From 933c913fbe6d2fbf277ff8d6b2f2298f0f84be64 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:30:59 +0200 Subject: [PATCH 14/15] Retain `None` as constraints bottom if no bottoms were provided (#19485) Current version replaces `None` (which indicates "no items") with an empty union (=`Uninhabited`). This breaks inference in some cases. Fixes #19483. --- mypy/solve.py | 6 ++++-- test-data/unit/check-recursive-types.test | 4 ++-- test-data/unit/check-typeddict.test | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mypy/solve.py b/mypy/solve.py index 098d926bc789..fbbcac2520ad 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -270,6 +270,7 @@ def solve_one(lowers: Iterable[Type], uppers: Iterable[Type]) -> Type | None: uppers = new_uppers # ...unless this is the only information we have, then we just pass it on. + lowers = list(lowers) if not uppers and not lowers: candidate = UninhabitedType() candidate.ambiguous = True @@ -281,10 +282,11 @@ def solve_one(lowers: Iterable[Type], uppers: Iterable[Type]) -> Type | None: # Process each bound separately, and calculate the lower and upper # bounds based on constraints. Note that we assume that the constraint # targets do not have constraint references. - if type_state.infer_unions: + if type_state.infer_unions and lowers: # This deviates from the general mypy semantics because # recursive types are union-heavy in 95% of cases. - bottom = UnionType.make_union(list(lowers)) + # Retain `None` when no bottoms were provided to avoid bogus `Never` inference. + bottom = UnionType.make_union(lowers) else: # The order of lowers is non-deterministic. # We attempt to sort lowers because joins are non-associative. For instance: diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 7ed5ea53c27e..86e9f02b5263 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -54,7 +54,7 @@ reveal_type(flatten([1, [2, [3]]])) # N: Revealed type is "builtins.list[builti class Bad: ... x: Nested[int] = [1, [2, [3]]] -x = [1, [Bad()]] # E: List item 1 has incompatible type "list[Bad]"; expected "Union[int, Nested[int]]" +x = [1, [Bad()]] # E: List item 0 has incompatible type "Bad"; expected "Union[int, Nested[int]]" [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasGenericInferenceNested] @@ -605,7 +605,7 @@ class NT(NamedTuple, Generic[T]): class A: ... class B(A): ... -nti: NT[int] = NT(key=0, value=NT(key=1, value=A())) # E: Argument "value" to "NT" has incompatible type "NT[A]"; expected "Union[int, NT[int]]" +nti: NT[int] = NT(key=0, value=NT(key=1, value=A())) # E: Argument "value" to "NT" has incompatible type "A"; expected "Union[int, NT[int]]" reveal_type(nti) # N: Revealed type is "tuple[builtins.int, Union[builtins.int, ...], fallback=__main__.NT[builtins.int]]" nta: NT[A] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index a068a63274ca..be5a6c655d8c 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -4271,3 +4271,21 @@ reveal_type(dicts.TF) # N: Revealed type is "def (*, user_id: builtins.int =) - reveal_type(dicts.TotalFalse) # N: Revealed type is "def (*, user_id: builtins.int =) -> TypedDict('__main__.Dicts.TF', {'user_id'?: builtins.int})" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testRecursiveNestedTypedDictInference] +from typing import TypedDict, Sequence +from typing_extensions import NotRequired + +class Component(TypedDict): + type: str + components: NotRequired[Sequence['Component']] + +inputs: Sequence[Component] = [{ + 'type': 'tuple', + 'components': [ + {'type': 'uint256'}, + {'type': 'address'}, + ] +}] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] From acb29831e286bbccde37c03bc75381f40a5fdc9e Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Wed, 30 Jul 2025 23:20:43 -0700 Subject: [PATCH 15/15] Bump version to 1.17.1 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index a305b237a771..d20fa047b8d6 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.17.1+dev" +__version__ = "1.17.1" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))