From cec1f7a40b508e8bc80949c060156c205f02ecb0 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 25 Apr 2025 21:52:14 +0500 Subject: [PATCH 1/8] Adjust c-analyzer max_sizes for typeobject.c --- Tools/c-analyzer/cpython/_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 037fe11ea223c7..682512801cd138 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -323,7 +323,7 @@ def clean_lines(text): _abs('Modules/_testcapimodule.c'): (20_000, 400), _abs('Modules/expat/expat.h'): (10_000, 400), _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), - _abs('Objects/typeobject.c'): (35_000, 200), + _abs('Objects/typeobject.c'): (380_000, 13_000), _abs('Python/compile.c'): (20_000, 500), _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), From 45be756b3dd96b5fd0902dab825badad4c28913c Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 00:18:18 +0500 Subject: [PATCH 2/8] Extend error messages if too much code to c-analyzer --- Tools/c-analyzer/c_parser/parser/__init__.py | 24 ++++++++++++++++++-- Tools/c-analyzer/c_parser/parser/_info.py | 10 ++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index ff4f303c4a2bec..e4a3d07a3c0d0f 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -208,7 +208,27 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): return # At this point either the file ended prematurely # or there's "too much" text. - filename, lno, text = srcinfo.filename, srcinfo._start, srcinfo.text + filename, lno_from, lno_to = srcinfo.filename, srcinfo.start, srcinfo.end + text = srcinfo.text if len(text) > 500: text = text[:500] + '...' - raise Exception(f'unmatched text ({filename} starting at line {lno}):\n{text}') + + if srcinfo.too_much_text(maxtext): + msg = [ + 'too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py', + f'{filename} starting at line {lno_from} to {lno_to}', + f'has code with length {len(text)} greater than {maxtext}:', + text + ] + raise Exception('\n'.join(msg)) + + if srcinfo.too_much_lines(maxlines): + msg = [ + 'too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py', + f'{filename} starting at line {lno_from} to {lno_to}', + f'has code with number of lines {lno_to - lno_from} greater than {maxlines}:', + text + ] + raise Exception('\n'.join(msg)) + + raise Exception(f'unmatched text ({filename} starting at line {lno_from}):\n{text}') diff --git a/Tools/c-analyzer/c_parser/parser/_info.py b/Tools/c-analyzer/c_parser/parser/_info.py index 340223db933c90..fbfc1d74f9a27a 100644 --- a/Tools/c-analyzer/c_parser/parser/_info.py +++ b/Tools/c-analyzer/c_parser/parser/_info.py @@ -123,10 +123,16 @@ def resolve(self, kind, data, name, parent=None): def done(self): self._set_ready() + def too_much_text(self, maxtext): + return maxtext and len(self.text) > maxtext + + def too_much_lines(self, maxlines): + return maxlines and self.end - self.start > maxlines + def too_much(self, maxtext, maxlines): - if maxtext and len(self.text) > maxtext: + if self.too_much_text(maxtext): pass - elif maxlines and self.end - self.start > maxlines: + elif self.too_much_lines(maxlines): pass else: return False From ce4b06cb8f4343de0f2c3cf80de3a69623ae1175 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 00:28:28 +0500 Subject: [PATCH 3/8] Make linter happy --- Tools/c-analyzer/c_parser/parser/_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_parser/parser/_info.py b/Tools/c-analyzer/c_parser/parser/_info.py index fbfc1d74f9a27a..3e831f977922a2 100644 --- a/Tools/c-analyzer/c_parser/parser/_info.py +++ b/Tools/c-analyzer/c_parser/parser/_info.py @@ -125,7 +125,7 @@ def done(self): def too_much_text(self, maxtext): return maxtext and len(self.text) > maxtext - + def too_much_lines(self, maxlines): return maxlines and self.end - self.start > maxlines From c514255ccd08574336db0c59f192c3abda0ee2fe Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 22 Jul 2025 22:24:08 +0500 Subject: [PATCH 4/8] Address review comments --- Tools/c-analyzer/c_parser/parser/__init__.py | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index e4a3d07a3c0d0f..bb3b9263aae457 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -214,21 +214,21 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): text = text[:500] + '...' if srcinfo.too_much_text(maxtext): - msg = [ - 'too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py', - f'{filename} starting at line {lno_from} to {lno_to}', - f'has code with length {len(text)} greater than {maxtext}:', - text - ] - raise Exception('\n'.join(msg)) + import textwrap + msg = f'''too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py + {filename} starting at line {lno_from} to {lno_to} + has code with length {len(text)} greater than {maxtext}: + {text} + ''' + raise RuntimeError(textwrap.dedent(msg)) if srcinfo.too_much_lines(maxlines): - msg = [ - 'too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py', - f'{filename} starting at line {lno_from} to {lno_to}', - f'has code with number of lines {lno_to - lno_from} greater than {maxlines}:', - text - ] - raise Exception('\n'.join(msg)) - - raise Exception(f'unmatched text ({filename} starting at line {lno_from}):\n{text}') + import textwrap + msg = f'''too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py + {filename} starting at line {lno_from} to {lno_to} + has code with number of lines {lno_to - lno_from} greater than {maxlines}: + {text} + ''' + raise RuntimeError(textwrap.dedent(msg)) + + raise RuntimeError(f'unmatched text ({filename} starting at line {lno_from}):\n{text}') From 4369d2d4107abc6593555659441645f7e58b870a Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 22 Jul 2025 22:34:40 +0500 Subject: [PATCH 5/8] Fix dedent usage --- Tools/c-analyzer/c_parser/parser/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index bb3b9263aae457..e1ceca9f32c030 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -215,7 +215,8 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): if srcinfo.too_much_text(maxtext): import textwrap - msg = f'''too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py + msg = f''' + too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to} has code with length {len(text)} greater than {maxtext}: {text} @@ -224,7 +225,8 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): if srcinfo.too_much_lines(maxlines): import textwrap - msg = f'''too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py + msg = f''' + too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to} has code with number of lines {lno_to - lno_from} greater than {maxlines}: {text} From e60d28c21e22341ec8d297106233b2bcfb5eda02 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 23 Jul 2025 10:00:48 +0500 Subject: [PATCH 6/8] Update Tools/c-analyzer/c_parser/parser/__init__.py Co-authored-by: Peter Bierma --- Tools/c-analyzer/c_parser/parser/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index e1ceca9f32c030..9ea4184e233846 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -226,7 +226,7 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): if srcinfo.too_much_lines(maxlines): import textwrap msg = f''' - too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py + too many lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to} has code with number of lines {lno_to - lno_from} greater than {maxlines}: {text} From 278f4d381e1f460e062c5e8e984c5cfae650fa2f Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 1 Aug 2025 23:31:45 +0500 Subject: [PATCH 7/8] Rename too_much_lines to too_many_lines --- Tools/c-analyzer/c_parser/parser/__init__.py | 2 +- Tools/c-analyzer/c_parser/parser/_info.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index e1ceca9f32c030..5aa7e79f3b661e 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -223,7 +223,7 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): ''' raise RuntimeError(textwrap.dedent(msg)) - if srcinfo.too_much_lines(maxlines): + if srcinfo.too_many_lines(maxlines): import textwrap msg = f''' too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py diff --git a/Tools/c-analyzer/c_parser/parser/_info.py b/Tools/c-analyzer/c_parser/parser/_info.py index 3e831f977922a2..13b192e2ce982f 100644 --- a/Tools/c-analyzer/c_parser/parser/_info.py +++ b/Tools/c-analyzer/c_parser/parser/_info.py @@ -126,13 +126,13 @@ def done(self): def too_much_text(self, maxtext): return maxtext and len(self.text) > maxtext - def too_much_lines(self, maxlines): + def too_many_lines(self, maxlines): return maxlines and self.end - self.start > maxlines def too_much(self, maxtext, maxlines): if self.too_much_text(maxtext): pass - elif self.too_much_lines(maxlines): + elif self.too_many_lines(maxlines): pass else: return False From 639e005e680ebd2cdd5daf01cab646c101f8d021 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sat, 2 Aug 2025 14:13:17 +0500 Subject: [PATCH 8/8] Move textwrap import to the top --- Tools/c-analyzer/c_parser/parser/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index f9b270f0031d44..f3f09107aefce6 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -116,6 +116,8 @@ * alt impl using a state machine (& tokenizer or split on delimiters) """ +import textwrap + from ..info import ParsedItem from ._info import SourceInfo @@ -214,7 +216,6 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): text = text[:500] + '...' if srcinfo.too_much_text(maxtext): - import textwrap msg = f''' too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to} @@ -224,7 +225,6 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): raise RuntimeError(textwrap.dedent(msg)) if srcinfo.too_many_lines(maxlines): - import textwrap msg = f''' too many lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to}