gh-92810: Reduce memory usage by ABCMeta.subclasscheck #131914

dolfinus · 2025-03-30T17:13:58Z

Issue: _abc._abc_subclasscheck has very poor performance and (I think) a memory leak #92810

For python build using --enable-optimizations:

test_performance_abc_isinstance.py

from abc import ABCMeta
#from _py_abc import ABCMeta
import time
import psutil
import os

class Root(metaclass=ABCMeta):
    pass

class Class1(Root):
    pass

class Class2(Root):
    pass

class Class3(Root):
    pass

class Class4(Root):
    subclasses = []
    @classmethod
    def __subclasses__(cls):
        return cls.subclasses


def _create_subclass1():
    class NestedClass1(Class1):
        pass

    return NestedClass1


def _create_subclass2():
    class NestedClass2(Class2):
        pass

    return NestedClass2


def _create_subclass3():
    class NestedClass3:
        pass

    Class3.register(NestedClass3)

    return NestedClass3


def _create_subclass4():
    class NestedClass4:
        pass

    Class4.subclasses.append(NestedClass4)

    return NestedClass4


def test_isinstance_performance():
    iters = 2000
    print("Creating new classes to check")
    objects_subclass1 = []
    for i in range(iters):
        subclass1 = _create_subclass1()
        objects_subclass1.append(subclass1())
    objects_subclass2 = []
    for i in range(iters):
        subclass2 = _create_subclass2()
        objects_subclass2.append(subclass2())

    objects_subclass3 = []
    for i in range(iters):
        subclass3 = _create_subclass3()
        objects_subclass3.append(subclass3())

    objects_subclass4 = []
    for i in range(iters):
        subclass4 = _create_subclass4()
        objects_subclass4.append(subclass4())

    # create one more subclass for sibling check
    another_subclass1 = _create_subclass1()
    another_subclass2 = _create_subclass2()
    another_subclass3 = _create_subclass3()
    another_subclass4 = _create_subclass4()

    print("Created subclasses =", len(objects_subclass1) + len(objects_subclass2) + len(objects_subclass3) + len(objects_subclass4))
    print("Consumed memory, Mb:", psutil.Process(os.getpid()).memory_info().rss / 1024 ** 2)
    print("Started compairing objects")

    # New subclass against parent, sibling, grantparent and cousin
    new_subclass1_vs_class1_ns = 0
    new_subclass1_vs_another_subclass1_ns = 0
    new_subclass1_vs_root_ns = 0
    new_subclass1_vs_class2_ns = 0
    for obj1 in objects_subclass1:
        start = time.perf_counter_ns()
        assert isinstance(obj1, Class1)
        new_subclass1_vs_class1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj1, another_subclass1)
        new_subclass1_vs_another_subclass1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert isinstance(obj1, Root)
        new_subclass1_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj1, Class2)
        new_subclass1_vs_class2_ns += time.perf_counter_ns() - start

    # Same subclass against parent, sibling, grantparent and cousin
    cached_subclass1_vs_root_ns = 0
    cached_subclass1_vs_another_subclass1_ns = 0
    cached_subclass1_vs_class1_ns = 0
    cached_subclass1_vs_class2_ns = 0
    for _ in range(iters):
        start = time.perf_counter_ns()
        assert isinstance(obj1, Class1)
        cached_subclass1_vs_class1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj1, another_subclass1)
        cached_subclass1_vs_another_subclass1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert isinstance(obj1, Root)
        cached_subclass1_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj1, Class2)
        cached_subclass1_vs_class2_ns += time.perf_counter_ns() - start

    # new class (via .register) against parent, sibling, grantparent and cousin
    new_subclass3_vs_class3_ns = 0
    new_subclass3_vs_another_subclass3_ns = 0
    new_subclass3_vs_root_ns = 0
    new_subclass3_vs_class1_ns = 0
    for obj3 in objects_subclass3:
        start = time.perf_counter_ns()
        assert isinstance(obj3, Class3)
        new_subclass3_vs_class3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj3, another_subclass3)
        new_subclass3_vs_another_subclass3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert isinstance(obj3, Root)
        new_subclass3_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj3, Class1)
        new_subclass3_vs_class1_ns += time.perf_counter_ns() - start

    # same class (via .register) against parent, sibling, grantparent and cousin
    cached_subclass3_vs_class3_ns = 0
    cached_subclass3_vs_another_subclass3_ns = 0
    cached_subclass3_vs_root_ns = 0
    cached_subclass3_vs_class1_ns = 0
    for _ in range(iters):
        start = time.perf_counter_ns()
        assert isinstance(obj3, Class3)
        cached_subclass3_vs_class3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj3, another_subclass3)
        cached_subclass3_vs_another_subclass3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert isinstance(obj3, Root)
        cached_subclass3_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj3, Class1)
        cached_subclass3_vs_class1_ns += time.perf_counter_ns() - start

    # new class (via __subclasses__) against parent, sibling, grantparent and cousin
    new_subclass4_vs_class4_ns = 0
    new_subclass4_vs_another_subclass4_ns = 0
    new_subclass4_vs_root_ns = 0
    new_subclass4_vs_class1_ns = 0
    for obj4 in objects_subclass4:
        start = time.perf_counter_ns()
        assert isinstance(obj4, Class4)
        new_subclass4_vs_class4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj4, another_subclass4)
        new_subclass4_vs_another_subclass4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert isinstance(obj4, Root)
        new_subclass4_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj4, Class1)
        new_subclass4_vs_class1_ns += time.perf_counter_ns() - start

    # same class (via __subclasses__) against parent, sibling, grantparent and cousin
    cached_subclass4_vs_class4_ns = 0
    cached_subclass4_vs_another_subclass4_ns = 0
    cached_subclass4_vs_root_ns = 0
    cached_subclass4_vs_class1_ns = 0
    for _ in range(iters):
        start = time.perf_counter_ns()
        assert isinstance(obj4, Class4)
        cached_subclass4_vs_class4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj4, another_subclass4)
        cached_subclass4_vs_another_subclass4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert isinstance(obj4, Root)
        cached_subclass4_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not isinstance(obj4, Class1)
        cached_subclass4_vs_class1_ns += time.perf_counter_ns() - start

    print("Completed compairing classes.")
    print("Consumed memory, Mb:", psutil.Process(os.getpid()).memory_info().rss / 1024 ** 2)

    total_ns = sum([
        cached_subclass1_vs_root_ns,
        cached_subclass1_vs_another_subclass1_ns,
        cached_subclass1_vs_class1_ns,
        cached_subclass1_vs_class2_ns,
        cached_subclass3_vs_class3_ns,
        cached_subclass3_vs_another_subclass3_ns,
        cached_subclass3_vs_root_ns,
        cached_subclass3_vs_class1_ns,
        cached_subclass4_vs_class4_ns,
        cached_subclass4_vs_another_subclass4_ns,
        cached_subclass4_vs_root_ns,
        cached_subclass4_vs_class1_ns,
        new_subclass1_vs_class1_ns,
        new_subclass1_vs_another_subclass1_ns,
        new_subclass1_vs_root_ns,
        new_subclass1_vs_class2_ns,
        new_subclass3_vs_class3_ns,
        new_subclass3_vs_another_subclass3_ns,
        new_subclass3_vs_root_ns,
        new_subclass3_vs_class1_ns,
        new_subclass4_vs_class4_ns,
        new_subclass4_vs_another_subclass4_ns,
        new_subclass4_vs_root_ns,
        new_subclass4_vs_class1_ns,
    ])
    print("Total, sec:", total_ns / 10**9)

    print("isinstance(cached class, parent), us:", cached_subclass1_vs_root_ns / iters / 1000)
    print("isinstance(cached class, sibling), us:", cached_subclass1_vs_another_subclass1_ns / iters / 1000)
    print("isinstance(cached class, grandparent), us:", cached_subclass1_vs_class1_ns / iters / 1000)
    print("isinstance(cached class, cousin), us:", cached_subclass1_vs_class2_ns / iters / 1000)

    print("isinstance(cached class, parent via .register()), us:", cached_subclass3_vs_class3_ns / iters / 1000)
    print("isinstance(cached class, sibling via .register()), us:", cached_subclass3_vs_another_subclass3_ns / iters / 1000)
    print("isinstance(cached class, grandparent via .register()), us:", cached_subclass3_vs_root_ns / iters / 1000)
    print("isinstance(cached class, cousin via .register()), us:", cached_subclass3_vs_class1_ns / iters / 1000)

    print("isinstance(cached class, parent via __subclasses__), us:", cached_subclass4_vs_class4_ns / iters / 1000)
    print("isinstance(cached class, sibling via __subclasses__), us:", cached_subclass4_vs_another_subclass4_ns / iters / 1000)
    print("isinstance(cached class, grandparent via __subclasses__), us:", cached_subclass4_vs_root_ns / iters / 1000)
    print("isinstance(cached class, cousin via __subclasses__), us:", cached_subclass4_vs_class1_ns / iters / 1000)

    print("isinstance(new class, parent), us:", new_subclass1_vs_class1_ns / len(objects_subclass1) / 1000)
    print("isinstance(new class, sibling), us:", new_subclass1_vs_another_subclass1_ns / len(objects_subclass1) / 1000)
    print("isinstance(new class, grandparent), us:", new_subclass1_vs_root_ns / len(objects_subclass1) / 1000)
    print("isinstance(new class, cousin), us:", new_subclass1_vs_class2_ns / len(objects_subclass1) / 1000)

    print("isinstance(new class, parent via .register()), us:", new_subclass3_vs_class3_ns / len(objects_subclass3) / 1000)
    print("isinstance(new class, sibling via .register()), us:", new_subclass3_vs_another_subclass3_ns / len(objects_subclass3) / 1000)
    print("isinstance(new class, grandparent via .register()), us:", new_subclass3_vs_root_ns / len(objects_subclass3) / 1000)
    print("isinstance(new class, cousin via .register()), us:", new_subclass3_vs_class1_ns / len(objects_subclass3) / 1000)

    print("isinstance(new class, parent via __subclasses__), us:", new_subclass4_vs_class4_ns / len(objects_subclass4) / 1000)
    print("isinstance(new class, sibling via __subclasses__), us:", new_subclass4_vs_another_subclass4_ns / len(objects_subclass4) / 1000)
    print("isinstance(new class, grandparent via __subclasses__), us:", new_subclass4_vs_root_ns / len(objects_subclass4) / 1000)
    print("isinstance(new class, cousin via __subclasses__), us:", new_subclass4_vs_class1_ns / len(objects_subclass4) / 1000)

if __name__ == "__main__":
    test_isinstance_performance()

test_performance_abc_issubclass.py

from abc import ABCMeta
#from _py_abc import ABCMeta
import time
import psutil
import os

class Root(metaclass=ABCMeta):
    pass

class Class1(Root):
    pass

class Class2(Root):
    pass

class Class3(Root):
    pass

class Class4(Root):
    subclasses = []
    @classmethod
    def __subclasses__(cls):
        return cls.subclasses


def _create_subclass1():
    class NestedClass1(Class1):
        pass

    return NestedClass1


def _create_subclass2():
    class NestedClass2(Class2):
        pass

    return NestedClass2


def _create_subclass3():
    class NestedClass3:
        pass

    Class3.register(NestedClass3)

    return NestedClass3


def _create_subclass4():
    class NestedClass4:
        pass

    Class4.subclasses.append(NestedClass4)

    return NestedClass4


def test_issubclass_performance():
    iters = 2000
    print("Creating new classes to check")
    nested_subclass1 = []
    for i in range(iters):
        subclass1 = _create_subclass1()
        nested_subclass1.append(subclass1)
    nested_subclass2 = []
    for i in range(iters):
        subclass2 = _create_subclass2()
        nested_subclass2.append(subclass2)

    nested_subclass3 = []
    for i in range(iters):
        subclass3 = _create_subclass3()
        nested_subclass3.append(subclass3)

    nested_subclass4 = []
    for i in range(iters):
        subclass4 = _create_subclass4()
        nested_subclass4.append(subclass4)

    # create one more subclass for sibling check
    another_subclass1 = _create_subclass1()
    another_subclass2 = _create_subclass2()
    another_subclass3 = _create_subclass3()
    another_subclass4 = _create_subclass4()

    print("Created subclasses =", len(nested_subclass1) + len(nested_subclass2) + len(nested_subclass3) + len(nested_subclass4))
    print("Consumed memory, Mb:", psutil.Process(os.getpid()).memory_info().rss / 1024 ** 2)
    print("Started compairing objects")

    # New subclass against parent, sibling, grantparent and cousin
    new_subclass1_vs_class1_ns = 0
    new_subclass1_vs_another_subclass1_ns = 0
    new_subclass1_vs_root_ns = 0
    new_subclass1_vs_class2_ns = 0
    for cls1 in nested_subclass1:
        start = time.perf_counter_ns()
        assert issubclass(cls1, Class1)
        new_subclass1_vs_class1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls1, another_subclass1)
        new_subclass1_vs_another_subclass1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert issubclass(cls1, Root)
        new_subclass1_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls1, Class2)
        new_subclass1_vs_class2_ns += time.perf_counter_ns() - start

    # Same subclass against parent, sibling, grantparent and cousin
    cached_subclass1_vs_root_ns = 0
    cached_subclass1_vs_another_subclass1_ns = 0
    cached_subclass1_vs_class1_ns = 0
    cached_subclass1_vs_class2_ns = 0
    for _ in range(iters):
        start = time.perf_counter_ns()
        assert issubclass(cls1, Class1)
        cached_subclass1_vs_class1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls1, another_subclass1)
        cached_subclass1_vs_another_subclass1_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert issubclass(cls1, Root)
        cached_subclass1_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls1, Class2)
        cached_subclass1_vs_class2_ns += time.perf_counter_ns() - start

    # new class (via .register) against parent, sibling, grantparent and cousin
    new_subclass3_vs_class3_ns = 0
    new_subclass3_vs_another_subclass3_ns = 0
    new_subclass3_vs_root_ns = 0
    new_subclass3_vs_class1_ns = 0
    for cls3 in nested_subclass3:
        start = time.perf_counter_ns()
        assert issubclass(cls3, Class3)
        new_subclass3_vs_class3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls3, another_subclass3)
        new_subclass3_vs_another_subclass3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert issubclass(cls3, Root)
        new_subclass3_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls3, Class1)
        new_subclass3_vs_class1_ns += time.perf_counter_ns() - start

    # same class (via .register) against parent, sibling, grantparent and cousin
    cached_subclass3_vs_class3_ns = 0
    cached_subclass3_vs_another_subclass3_ns = 0
    cached_subclass3_vs_root_ns = 0
    cached_subclass3_vs_class1_ns = 0
    for _ in range(iters):
        start = time.perf_counter_ns()
        assert issubclass(cls3, Class3)
        cached_subclass3_vs_class3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls3, another_subclass3)
        cached_subclass3_vs_another_subclass3_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert issubclass(cls3, Root)
        cached_subclass3_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls3, Class1)
        cached_subclass3_vs_class1_ns += time.perf_counter_ns() - start

    # new class (via __subclasses__) against parent, sibling, grantparent and cousin
    new_subclass4_vs_class4_ns = 0
    new_subclass4_vs_another_subclass4_ns = 0
    new_subclass4_vs_root_ns = 0
    new_subclass4_vs_class1_ns = 0
    for cls4 in nested_subclass4:
        start = time.perf_counter_ns()
        assert issubclass(cls4, Class4)
        new_subclass4_vs_class4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls4, another_subclass4)
        new_subclass4_vs_another_subclass4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert issubclass(cls4, Root)
        new_subclass4_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls4, Class1)
        new_subclass4_vs_class1_ns += time.perf_counter_ns() - start

    # same class (via __subclasses__) against parent, sibling, grantparent and cousin
    cached_subclass4_vs_class4_ns = 0
    cached_subclass4_vs_another_subclass4_ns = 0
    cached_subclass4_vs_root_ns = 0
    cached_subclass4_vs_class1_ns = 0
    for _ in range(iters):
        start = time.perf_counter_ns()
        assert issubclass(cls4, Class4)
        cached_subclass4_vs_class4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls4, another_subclass4)
        cached_subclass4_vs_another_subclass4_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert issubclass(cls4, Root)
        cached_subclass4_vs_root_ns += time.perf_counter_ns() - start

        start = time.perf_counter_ns()
        assert not issubclass(cls4, Class1)
        cached_subclass4_vs_class1_ns += time.perf_counter_ns() - start

    print("Completed compairing classes.")
    print("Consumed memory, Mb:", psutil.Process(os.getpid()).memory_info().rss / 1024 ** 2)

    total_ns = sum([
        cached_subclass1_vs_root_ns,
        cached_subclass1_vs_another_subclass1_ns,
        cached_subclass1_vs_class1_ns,
        cached_subclass1_vs_class2_ns,
        cached_subclass3_vs_class3_ns,
        cached_subclass3_vs_another_subclass3_ns,
        cached_subclass3_vs_root_ns,
        cached_subclass3_vs_class1_ns,
        cached_subclass4_vs_class4_ns,
        cached_subclass4_vs_another_subclass4_ns,
        cached_subclass4_vs_root_ns,
        cached_subclass4_vs_class1_ns,
        new_subclass1_vs_class1_ns,
        new_subclass1_vs_another_subclass1_ns,
        new_subclass1_vs_root_ns,
        new_subclass1_vs_class2_ns,
        new_subclass3_vs_class3_ns,
        new_subclass3_vs_another_subclass3_ns,
        new_subclass3_vs_root_ns,
        new_subclass3_vs_class1_ns,
        new_subclass4_vs_class4_ns,
        new_subclass4_vs_another_subclass4_ns,
        new_subclass4_vs_root_ns,
        new_subclass4_vs_class1_ns,
    ])
    print("Total, sec:", total_ns / 10**9)

    print("issubclass(cached class, parent), us:", cached_subclass1_vs_root_ns / iters / 1000)
    print("issubclass(cached class, sibling), us:", cached_subclass1_vs_another_subclass1_ns / iters / 1000)
    print("issubclass(cached class, grandparent), us:", cached_subclass1_vs_class1_ns / iters / 1000)
    print("issubclass(cached class, cousin), us:", cached_subclass1_vs_class2_ns / iters / 1000)

    print("issubclass(cached class, parent via .register()), us:", cached_subclass3_vs_class3_ns / iters / 1000)
    print("issubclass(cached class, sibling via .register()), us:", cached_subclass3_vs_another_subclass3_ns / iters / 1000)
    print("issubclass(cached class, grandparent via .register()), us:", cached_subclass3_vs_root_ns / iters / 1000)
    print("issubclass(cached class, cousin via .register()), us:", cached_subclass3_vs_class1_ns / iters / 1000)

    print("issubclass(cached class, parent via __subclasses__), us:", cached_subclass4_vs_class4_ns / iters / 1000)
    print("issubclass(cached class, sibling via __subclasses__), us:", cached_subclass4_vs_another_subclass4_ns / iters / 1000)
    print("issubclass(cached class, grandparent via __subclasses__), us:", cached_subclass4_vs_root_ns / iters / 1000)
    print("issubclass(cached class, cousin via __subclasses__), us:", cached_subclass4_vs_class1_ns / iters / 1000)

    print("issubclass(new class, parent), us:", new_subclass1_vs_class1_ns / len(nested_subclass1) / 1000)
    print("issubclass(new class, sibling), us:", new_subclass1_vs_another_subclass1_ns / len(nested_subclass1) / 1000)
    print("issubclass(new class, grandparent), us:", new_subclass1_vs_root_ns / len(nested_subclass1) / 1000)
    print("issubclass(new class, cousin), us:", new_subclass1_vs_class2_ns / len(nested_subclass1) / 1000)

    print("issubclass(new class, parent via .register()), us:", new_subclass3_vs_class3_ns / len(nested_subclass3) / 1000)
    print("issubclass(new class, sibling via .register()), us:", new_subclass3_vs_another_subclass3_ns / len(nested_subclass3) / 1000)
    print("issubclass(new class, grandparent via .register()), us:", new_subclass3_vs_root_ns / len(nested_subclass3) / 1000)
    print("issubclass(new class, cousin via .register()), us:", new_subclass3_vs_class1_ns / len(nested_subclass3) / 1000)

    print("issubclass(new class, parent via __subclasses__), us:", new_subclass4_vs_class4_ns / len(nested_subclass4) / 1000)
    print("issubclass(new class, sibling via __subclasses__), us:", new_subclass4_vs_another_subclass4_ns / len(nested_subclass4) / 1000)
    print("issubclass(new class, grandparent via __subclasses__), us:", new_subclass4_vs_root_ns / len(nested_subclass4) / 1000)
    print("issubclass(new class, cousin via __subclasses__), us:", new_subclass4_vs_class1_ns / len(nested_subclass4) / 1000)

if __name__ == "__main__":
    test_issubclass_performance()

For 8k nested subclasses:

Impl	Memory before, MB	Memory after, MB
_abc	4365.906	42.313
_py_abc	2839.828	48.434

Impl	Total time before, seconds	Total time after, seconds
_abc	484.174	29.418
_py_abc	189.669	91.173

Check	Impl	us per check, before	us per check, after	Impl	us per check, before	us per check, after
`isinstance(cached class, parent)`	_abc	0.315	0.344	_py_abc	0.614	0.518
`isinstance(cached class, sibling)`	_abc	0.317	0.342	_py_abc	1.074	0.867
`isinstance(cached class, grandparent)`	_abc	0.319	0.334	_py_abc	0.642	0.525
`isinstance(cached class, cousin)`	_abc	0.302	0.327	_py_abc	1.034	0.868
`isinstance(cached class, parent via .register())`	_abc	1.095	0.663	_py_abc	0.621	0.620
`isinstance(cached class, sibling via .register())`	_abc	0.272	0.183	_py_abc	0.176	0.166
`isinstance(cached class, grandparent via .register())`	_abc	0.599	0.376	_py_abc	0.573	0.605
`isinstance(cached class, cousin via .register())`	_abc	0.632	0.413	_py_abc	0.988	1.071
`isinstance(cached class, parent via __subclasses__)`	_abc	0.360	0.352	_py_abc	0.583	0.513
`isinstance(cached class, sibling via __subclasses__)`	_abc	0.170	0.158	_py_abc	0.177	0.137
`isinstance(cached class, grandparent via __subclasses__)`	_abc	0.350	0.326	_py_abc	0.557	0.480
`isinstance(cached class, cousin via __subclasses__)`	_abc	0.375	0.351	_py_abc	0.959	0.888
`isinstance(new class, parent)`	_abc	4.319	5.937	_py_abc	8.072	5.913
`isinstance(new class, sibling)`	_abc	3.021	4.142	_py_abc	6.139	5.906
`isinstance(new class, grandparent)`	_abc	1.771	2.091	_py_abc	4.068	3.269
`isinstance(new class, cousin)`	_abc	10_074.456	2_171.508	_py_abc	8_975.521	4_839.700
`isinstance(new class, parent via .register())`	_abc	2.490	5.037	_py_abc	307.863	263.306
`isinstance(new class, sibling via .register())`	_abc	0.731	0.717	_py_abc	0.954	0.840
`isinstance(new class, grandparent via .register())`	_abc	78_883.538	3_892.012	_py_abc	30_999.150	10_110.654
`isinstance(new class, cousin via .register())`	_abc	3.770	1_965.657	_py_abc	4.778	4_995.907
`isinstance(new class, parent via __subclasses__)`	_abc	48.480	233.732	_py_abc	110.832	170.838
`isinstance(new class, sibling via __subclasses__)`	_abc	0.841	1.155	_py_abc	0.774	0.589
`isinstance(new class, grandparent via __subclasses__)`	_abc	153_055.472	4_369.760	_py_abc	54_403.267	9_811.822
`isinstance(new class, cousin via __subclasses__)`	_abc	3.035	2_053.119	_py_abc	5.259	4_685.862

Check	Impl	us per check, before	us per check, after	Impl	us per check, before	us per check, after
`issubclass(cached class, parent)`	_abc	0.316	0.295	_py_abc	0.546	0.503
`issubclass(cached class, sibling)`	_abc	0.302	0.289	_py_abc	0.909	0.841
`issubclass(cached class, grandparent)`	_abc	0.335	0.292	_py_abc	0.571	0.526
`issubclass(cached class, cousin)`	_abc	0.299	0.289	_py_abc	0.889	0.839
`issubclass(cached class, parent via .register())`	_abc	0.594	0.343	_py_abc	0.586	0.515
`issubclass(cached class, sibling via .register())`	_abc	0.224	0.121	_py_abc	0.141	0.125
`issubclass(cached class, grandparent via .register())`	_abc	0.456	0.287	_py_abc	0.548	0.495
`issubclass(cached class, cousin via .register())`	_abc	0.496	0.291	_py_abc	0.884	0.876
`issubclass(cached class, parent via __subclasses__)`	_abc	0.459	0.354	_py_abc	0.588	0.538
`issubclass(cached class, sibling via __subclasses__)`	_abc	0.185	0.141	_py_abc	0.150	0.129
`issubclass(cached class, grandparent via __subclasses__)`	_abc	0.453	0.320	_py_abc	0.554	0.508
`issubclass(cached class, cousin via __subclasses__)`	_abc	0.452	0.339	_py_abc	0.945	0.897
`issubclass(new class, parent)`	_abc	2.571	2.586	_py_abc	3.999	3.313
`issubclass(new class, sibling)`	_abc	2.325	2.517	_py_abc	4.007	3.135
`issubclass(new class, grandparent)`	_abc	1.262	1.318	_py_abc	2.790	2.441
`issubclass(new class, cousin)`	_abc	10_104.771	1_680.304	_py_abc	8_350.500	14_576.658
`issubclass(new class, parent via .register())`	_abc	0.993	3.510	_py_abc	313.716	266.526
`issubclass(new class, sibling via .register())`	_abc	0.291	0.329	_py_abc	0.317	0.345
`issubclass(new class, grandparent via .register())`	_abc	71_022.606	3_882.094	_py_abc	32_358.990	39_985.259
`issubclass(new class, cousin via .register())`	_abc	1.753	1_984.566	_py_abc	2.398	14_927.480
`issubclass(new class, parent via __subclasses__)`	_abc	61.173	200.559	_py_abc	95.513	169.280
`issubclass(new class, sibling via __subclasses__)`	_abc	0.506	0.463	_py_abc	0.274	0.306
`issubclass(new class, grandparent via __subclasses__)`	_abc	164_455.899	3_411.897	_py_abc	49_661.247	310_012.451
`issubclass(new class, cousin via __subclasses__)`	_abc	2.554	1_586.308	_py_abc	2.196	14_798.255

bedevere-app · 2025-03-30T17:14:02Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

Lib/_py_abc.py

Modules/_abc.c

Lib/test/test_abstract_numbers.py

Lib/test/test_abc.py

bedevere-app · 2025-03-31T11:24:02Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

bedevere-app · 2025-03-31T11:25:30Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

bedevere-app · 2025-03-31T11:26:44Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

bedevere-app · 2025-03-31T11:29:01Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

Lib/test/test_abc.py

Lib/_py_abc.py

Lib/test/test_isinstance.py

python-cla-bot · 2025-04-06T13:56:20Z

All commit authors signed the Contributor License Agreement.

bedevere-app · 2025-04-21T10:57:37Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

Signed-off-by: Martynov Maxim <martinov_m_s_@mail.ru>

bedevere-app · 2025-04-21T11:04:00Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

bedevere-app · 2025-04-21T11:05:29Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

bedevere-app · 2025-04-21T11:18:13Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

bedevere-app · 2025-06-13T14:09:23Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

bedevere-app · 2025-06-13T14:14:44Z

Most changes to Python require a NEWS entry. Add one using the blurb_it web app or the blurb command-line tool.

If this change has little impact on Python users, wait for a maintainer to apply the skip news label instead.

Modules/_abc.c

Viicos · 2025-07-01T10:04:56Z

Do you happen to know this will play with #119719?

dolfinus · 2025-07-01T10:12:32Z

Do you happen to know this will play with #119719?

No, and I don't like solution of #119719 at all

dolfinus · 2025-07-18T18:06:20Z

@picnixz Could you please take a look on this PR?

Lib/test/test_abc.py

Misc/NEWS.d/next/Core_and_Builtins/2025-06-13-17-22-34.gh-issue-92810.Tb6x6C.rst

Modules/_abc.c

Lib/_py_abc.py

Lib/test/test_abc.py

Modules/_abc.c

picnixz · 2025-08-04T11:45:46Z

Did the improvements in subclass checks actually introduced improvements in isinstance checks right? (because your benchmarks are all with respect to isinstance, not issubclass). If not, please update the benchmarks to indicate the improvements of issubclass itself.

dolfinus · 2025-08-04T11:51:21Z

because your benchmarks are all with respect to isinstance, not issubclass

This is because in real applications isinstance is much more abundant than issubclass. For example, pydantic checks that object is already an instance of annotated class, and skips validation in this case.

I can add both cases to benchmark, if required.

picnixz · 2025-08-04T12:44:51Z

Yes, please add both benchmarks.

dolfinus · 2025-08-07T14:27:14Z

Added scripts and microbenchmark results to PR description

bedevere-app bot mentioned this pull request Mar 30, 2025

_abc._abc_subclasscheck has very poor performance and (I think) a memory leak #92810

Open

picnixz reviewed Mar 30, 2025

View reviewed changes

picnixz reviewed Apr 1, 2025

View reviewed changes

Lib/test/test_abc.py Outdated Show resolved Hide resolved

Lib/test/test_abc.py Outdated Show resolved Hide resolved

Lib/_py_abc.py Outdated Show resolved Hide resolved

picnixz reviewed Apr 1, 2025

View reviewed changes

Lib/test/test_isinstance.py Outdated Show resolved Hide resolved

Lib/test/test_isinstance.py Outdated Show resolved Hide resolved

Viicos mentioned this pull request Apr 1, 2025

Warn if registering virtual subclasses on Pydantic models pydantic/pydantic#11669

Draft

5 tasks

dolfinus added 6 commits April 21, 2025 14:03

pythongh-92810: Avoid O(n^2) complexity in ABCMeta.__subclasscheck__

675bec5

Signed-off-by: Martynov Maxim <martinov_m_s_@mail.ru>

pythongh-92810: Apply fixes

701ecc9

Signed-off-by: Martynov Maxim <martinov_m_s_@mail.ru>

pythongh-92810: Apply fixes

041f109

Signed-off-by: Martynov Maxim <martinov_m_s_@mail.ru>

pythongh-92810: Apply fixes

9bc4385

Signed-off-by: Martynov Maxim <martinov_m_s_@mail.ru>

pythongh-92810: Apply fixes

3d80b1e

Signed-off-by: Martynov Maxim <martinov_m_s_@mail.ru>

pythongh-92810: Return __subclasses__clause back

b7603e0

dolfinus force-pushed the improvement/ABCMeta_subclasscheck branch from abf4bfe to b7603e0 Compare April 21, 2025 11:03

pythongh-92810: Revert _abc.c changes

dd0d18c

pythongh-92810: Fix linter errors

8d695fd

dolfinus changed the title ~~gh-92810: Avoid O(n^2) complexity in ABCMeta.__subclasscheck__~~ gh-92810: Reduce memory usage by ABCMeta.__subclasscheck__ Apr 23, 2025

Viicos mentioned this pull request Apr 29, 2025

pydantic using abc which can have memory and performance issues pydantic/pydantic#11605

Open

1 task

pythongh-92810: Add recursive issubclass check to _abc.c

a2650b6

pythongh-92810: Remove WeakKeyDictionary from _py_abc

7afa5ea

dolfinus added 2 commits June 22, 2025 19:13

pythongh-92810: Fixes after review

6fc994d

pythongh-92810: Fixes after review

b3b5895

dolfinus requested a review from picnixz June 22, 2025 16:42

ZeroIntensity reviewed Jun 22, 2025

View reviewed changes

Modules/_abc.c Outdated Show resolved Hide resolved

dolfinus added 5 commits June 23, 2025 18:53

pythongh-92810: Fixes after review

69c5038

pythongh-92810: Fixes after review

dc1b6d5

pythongh-92810: Introduce FT wrappers for uint64_t atomics

cd097ab

pythongh-92810: Use FT atomic wrappers for ABC invalidation counter

f3a21a7

pythongh-92810: Fix missing FT wrapper

e24e815

picnixz reviewed Aug 2, 2025

View reviewed changes

dolfinus added 2 commits August 4, 2025 11:35

pythongh-92810: Address review fixes

b723912

Merge branch 'main' into improvement/ABCMeta_subclasscheck

0295846

picnixz reviewed Aug 4, 2025

View reviewed changes

Modules/_abc.c Show resolved Hide resolved

Modules/_abc.c Show resolved Hide resolved

Modules/_abc.c Show resolved Hide resolved

dolfinus added 3 commits August 4, 2025 13:00

pythongh-92810: Address review fixes

16f39bd

pythongh-92810: Add What's New entry

2dc6453

pythongh-92810: Fix What's New entry syntax

968766d

picnixz reviewed Aug 4, 2025

View reviewed changes

Lib/_py_abc.py Outdated Show resolved Hide resolved

Lib/test/test_abc.py Outdated Show resolved Hide resolved

Modules/_abc.c Show resolved Hide resolved

dolfinus added 3 commits August 4, 2025 14:34

pythongh-92810: Address review fixes

a6e4461

pythongh-92810: Address review fixes

80d3281

pythongh-92810: Properly reset recursion check

ff38b9e

Merge branch 'main' into improvement/ABCMeta_subclasscheck

23df287

Merge branch 'main' into improvement/ABCMeta_subclasscheck

0bf7374

dolfinus requested a review from AA-Turner as a code owner August 7, 2025 11:38

Uh oh!

gh-92810: Reduce memory usage by ABCMeta.__subclasscheck__ #131914

Are you sure you want to change the base?

gh-92810: Reduce memory usage by ABCMeta.__subclasscheck__ #131914

Uh oh!

Conversation

dolfinus commented Mar 30, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

bedevere-app bot commented Mar 30, 2025

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

bedevere-app bot commented Mar 31, 2025

Uh oh!

bedevere-app bot commented Mar 31, 2025

Uh oh!

bedevere-app bot commented Mar 31, 2025

Uh oh!

bedevere-app bot commented Mar 31, 2025

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

python-cla-bot bot commented Apr 6, 2025

Uh oh!

bedevere-app bot commented Apr 21, 2025

Uh oh!

bedevere-app bot commented Apr 21, 2025

Uh oh!

bedevere-app bot commented Apr 21, 2025

Uh oh!

bedevere-app bot commented Apr 21, 2025

Uh oh!

bedevere-app bot commented Jun 13, 2025

Uh oh!

bedevere-app bot commented Jun 13, 2025

Uh oh!

Uh oh!

Viicos commented Jul 1, 2025

Uh oh!

dolfinus commented Jul 1, 2025

Uh oh!

dolfinus commented Jul 18, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

picnixz commented Aug 4, 2025

Uh oh!

dolfinus commented Aug 4, 2025

Uh oh!

picnixz commented Aug 4, 2025

Uh oh!

dolfinus commented Aug 7, 2025

Uh oh!

Uh oh!

gh-92810: Reduce memory usage by ABCMeta.subclasscheck #131914

gh-92810: Reduce memory usage by ABCMeta.subclasscheck #131914

dolfinus commented Mar 30, 2025 •

edited

Loading

dolfinus commented Jul 18, 2025 •

edited

Loading