From d898d27dad53846a7943a818e4010a49a53c3895 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Fri, 4 Apr 2025 20:24:57 -0700 Subject: [PATCH 1/4] gh-132108: Add Buffer Protocol support to int.from_bytes Speed up conversion from `bytes-like` objects like `bytearray` while keeping conversion from `bytes` stable. On a `--with-lto --enable-optimizaitons` build on my 64 bit Linux box: new: from_bytes_flags: Mean +- std dev: 28.6 ns +- 0.5 ns bench_convert[bytes]: Mean +- std dev: 50.4 ns +- 1.4 ns bench_convert[bytearray]: Mean +- std dev: 51.3 ns +- 0.7 ns old: from_bytes_flags: Mean +- std dev: 28.1 ns +- 1.1 ns bench_convert[bytes]: Mean +- std dev: 50.3 ns +- 4.3 ns bench_convert[bytearray]: Mean +- std dev: 64.7 ns +- 0.9 ns Benchmark code: ```python import pyperf import time def from_bytes_flags(loops): range_it = range(loops) t0 = time.perf_counter() for _ in range_it: int.from_bytes(b'\x00\x10', byteorder='big') int.from_bytes(b'\x00\x10', byteorder='little') int.from_bytes(b'\xfc\x00', byteorder='big', signed=True) int.from_bytes(b'\xfc\x00', byteorder='big', signed=False) int.from_bytes([255, 0, 0], byteorder='big') return time.perf_counter() - t0 sample_bytes = [ b'', b'\x00', b'\x01', b'\x7f', b'\x80', b'\xff', b'\x01\x00', b'\x7f\xff', b'\x80\x00', b'\xff\xff', b'\x01\x00\x00', ] sample_bytearray = [bytearray(v) for v in sample_bytes] def bench_convert(loops, values): range_it = range(loops) t0 = time.perf_counter() for _ in range_it: for val in values: int.from_bytes(val) return time.perf_counter() - t0 runner = pyperf.Runner() runner.bench_time_func('from_bytes_flags', from_bytes_flags, inner_loops=10) runner.bench_time_func('bench_convert[bytes]', bench_convert, sample_bytes, inner_loops=10) runner.bench_time_func('bench_convert[bytearray]', bench_convert, sample_bytearray, inner_loops=10) ``` --- ...-04-04-20-38-29.gh-issue-132108.UwZIQy.rst | 2 ++ Objects/longobject.c | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst new file mode 100644 index 00000000000000..a4609b34abe416 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst @@ -0,0 +1,2 @@ +Speed up :meth:`int.from_bytes` when passed a bytes-like object such as a +:class:`bytearray`. diff --git a/Objects/longobject.c b/Objects/longobject.c index 692312c1ad976c..a8ac7524d70f71 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6439,6 +6439,7 @@ int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, { int little_endian; PyObject *long_obj, *bytes; + Py_buffer view; if (byteorder == NULL) little_endian = 0; @@ -6452,14 +6453,26 @@ int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, return NULL; } - bytes = PyObject_Bytes(bytes_obj); - if (bytes == NULL) - return NULL; - - long_obj = _PyLong_FromByteArray( - (unsigned char *)PyBytes_AS_STRING(bytes), Py_SIZE(bytes), - little_endian, is_signed); - Py_DECREF(bytes); + /* Use buffer protocol to avoid copies. */ + if (PyObject_CheckBuffer(bytes_obj)) { + if (PyObject_GetBuffer(bytes_obj, &view, PyBUF_SIMPLE) != 0) { + return NULL; + } + long_obj = _PyLong_FromByteArray(view.buf, view.len, little_endian, + is_signed); + PyBuffer_Release(&view); + } + else { + /* fallback: Construct a bytes then convert. */ + bytes = PyObject_Bytes(bytes_obj); + if (bytes == NULL) { + return NULL; + } + long_obj = _PyLong_FromByteArray( + (unsigned char *)PyBytes_AS_STRING(bytes), Py_SIZE(bytes), + little_endian, is_signed); + Py_DECREF(bytes); + } if (long_obj != NULL && type != &PyLong_Type) { Py_SETREF(long_obj, PyObject_CallOneArg((PyObject *)type, long_obj)); From b078ea1ea0a1568e74d5688e41ea2af469392a65 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 7 Apr 2025 11:59:47 -0700 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Sergey B Kirpichev --- .../2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst index a4609b34abe416..99f319517b6472 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst @@ -1,2 +1,2 @@ -Speed up :meth:`int.from_bytes` when passed a bytes-like object such as a -:class:`bytearray`. +Speed up :meth:`int.from_bytes` when passed object supports :ref:`buffer +protocol `, like :class:`bytearray`. From e6ef5e159e6265634a4a9422e7048e7e251b954e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 8 Apr 2025 20:03:13 -0700 Subject: [PATCH 3/4] Check bytes exact first to speed up that case from_bytes_flags: Mean +- std dev: [main] 28.3 ns +- 1.3 ns -> [exactbytes] 27.3 ns +- 0.3 ns: 1.04x faster bench_convert[bytearray]: Mean +- std dev: [main] 65.8 ns +- 3.3 ns -> [exactbytes] 53.1 ns +- 5.1 ns: 1.24x faster bench_convert_big[bytes]: Mean +- std dev: [main] 51.8 ns +- 0.6 ns -> [exactbytes] 50.3 ns +- 0.5 ns: 1.03x faster bench_convert_big[bytearray]: Mean +- std dev: [main] 65.8 ns +- 3.0 ns -> [exactbytes] 53.5 ns +- 5.3 ns: 1.23x faster Benchmark hidden because not significant (1): bench_convert[bytes] --- .../2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst | 2 +- Objects/longobject.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst index 99f319517b6472..8c2d947954c5bb 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-04-20-38-29.gh-issue-132108.UwZIQy.rst @@ -1,2 +1,2 @@ Speed up :meth:`int.from_bytes` when passed object supports :ref:`buffer -protocol `, like :class:`bytearray`. +protocol `, like :class:`bytearray` by ~1.2x. diff --git a/Objects/longobject.c b/Objects/longobject.c index a8ac7524d70f71..a0f7a2e9cecadc 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6454,7 +6454,11 @@ int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, } /* Use buffer protocol to avoid copies. */ - if (PyObject_CheckBuffer(bytes_obj)) { + if (PyBytes_CheckExact(bytes_obj)) { + long_obj = _PyLong_FromByteArray( + (unsigned char *)PyBytes_AS_STRING(bytes_obj), Py_SIZE(bytes_obj), + little_endian, is_signed); + } else if (PyObject_CheckBuffer(bytes_obj)) { if (PyObject_GetBuffer(bytes_obj, &view, PyBUF_SIMPLE) != 0) { return NULL; } From 9c511b15d0b4e717dacc6170f377b9cd23eeca79 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 21 Apr 2025 23:47:18 -0700 Subject: [PATCH 4/4] Tweak comments, fix PEP-7 formatting --- Objects/longobject.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index a0f7a2e9cecadc..52aa65290fa072 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6453,12 +6453,14 @@ int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, return NULL; } - /* Use buffer protocol to avoid copies. */ + /* Fast-path exact bytes. */ if (PyBytes_CheckExact(bytes_obj)) { long_obj = _PyLong_FromByteArray( (unsigned char *)PyBytes_AS_STRING(bytes_obj), Py_SIZE(bytes_obj), little_endian, is_signed); - } else if (PyObject_CheckBuffer(bytes_obj)) { + } + /* Use buffer protocol to avoid copies. */ + else if (PyObject_CheckBuffer(bytes_obj)) { if (PyObject_GetBuffer(bytes_obj, &view, PyBUF_SIMPLE) != 0) { return NULL; }