From 5be56a065161687211243d0399a74b01cc7439a3 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Tue, 28 May 2024 17:38:17 +0100 Subject: [PATCH 01/11] Simplify existing control flow logic of shlex.quote Removes early return for the special case of an empty string --- Lib/shlex.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Lib/shlex.py b/Lib/shlex.py index f4821616b62a0f..14a19460af541b 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -322,15 +322,12 @@ def join(split_command): def quote(s): """Return a shell-escaped version of the string *s*.""" - if not s: - return "''" - if _find_unsafe(s) is None: - return s - - # use single quotes, and put single quotes into double quotes - # the string $'b is then quoted as '$'"'"'b' - return "'" + s.replace("'", "'\"'\"'") + "'" + if not s or _find_unsafe(s) is not None: + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + return s def _print_tokens(lexer): while tt := lexer.get_token(): From 1305d906712fba1ed60967eeeb23749aed9de187 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Tue, 28 May 2024 17:40:20 +0100 Subject: [PATCH 02/11] Add 'always' kwarg to shlex.quote --- Lib/shlex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/shlex.py b/Lib/shlex.py index 14a19460af541b..db6f1356c5e104 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -320,9 +320,9 @@ def join(split_command): _find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search -def quote(s): +def quote(s, always=False): """Return a shell-escaped version of the string *s*.""" - if not s or _find_unsafe(s) is not None: + if always or not s or _find_unsafe(s) is not None: # use single quotes, and put single quotes into double quotes # the string $'b is then quoted as '$'"'"'b' return "'" + s.replace("'", "'\"'\"'") + "'" From 9748cc17b4b2d1b34b5995a480c950ec9741c7cd Mon Sep 17 00:00:00 2001 From: jb2170 Date: Tue, 28 May 2024 18:08:00 +0100 Subject: [PATCH 03/11] Update shlex docs for shlex.quote, adding info about the always kwarg --- Doc/library/shlex.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index a96f0864dc1260..edfe2071fcfcea 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -49,11 +49,13 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.8 -.. function:: quote(s) +.. function:: quote(s, always=False) Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for - cases where you cannot use a list. + cases where you cannot use a list. If the *always* keyword argument is + set to ``True`` then the string *s* will always be escaped, even in the + absence of special characters. .. _shlex-quote-warning: @@ -98,6 +100,9 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.3 + .. versionchanged:: 3.14 + Added the *always* parameter. + The :mod:`shlex` module defines the following class: From 0048c87cc904daeccf665fa492b8e33b050e6845 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 17:24:19 +0000 Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst b/Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst new file mode 100644 index 00000000000000..6fd3dfd2042e42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst @@ -0,0 +1,2 @@ +:func:`shlex.quote` now has an *always* keyword argument for forcing +the escaping of the string passed to it From 36be111edab30efb36df553e0ab51dba4c8b3615 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Mon, 3 Jun 2024 23:15:10 +0100 Subject: [PATCH 05/11] Keep the empty string clearly a special case for shlex.quote --- Lib/shlex.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/shlex.py b/Lib/shlex.py index db6f1356c5e104..cf59d806286387 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -322,7 +322,10 @@ def join(split_command): def quote(s, always=False): """Return a shell-escaped version of the string *s*.""" - if always or not s or _find_unsafe(s) is not None: + if not s: + return "''" + + if always or _find_unsafe(s) is not None: # use single quotes, and put single quotes into double quotes # the string $'b is then quoted as '$'"'"'b' return "'" + s.replace("'", "'\"'\"'") + "'" From 9e9eff0047d731a42f972e5ee20eec6d30503a30 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Mon, 3 Jun 2024 23:49:36 +0100 Subject: [PATCH 06/11] Make `always` a keyword-only argument for shlex.quote --- Doc/library/shlex.rst | 2 +- Lib/shlex.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index edfe2071fcfcea..3f9887857344ca 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -49,7 +49,7 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.8 -.. function:: quote(s, always=False) +.. function:: quote(s, *, always=False) Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for diff --git a/Lib/shlex.py b/Lib/shlex.py index cf59d806286387..c5c5c0ef420921 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -320,7 +320,7 @@ def join(split_command): _find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search -def quote(s, always=False): +def quote(s, *, always=False): """Return a shell-escaped version of the string *s*.""" if not s: return "''" From 5f74da5d9ec1a24b2663364a974e70cb257016ed Mon Sep 17 00:00:00 2001 From: jb2170 Date: Mon, 3 Jun 2024 23:50:11 +0100 Subject: [PATCH 07/11] Update shlex docs explaining reasoning for `quote(..., always=True)` --- Doc/library/shlex.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 3f9887857344ca..408b0cf137755c 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -53,9 +53,16 @@ The :mod:`shlex` module defines the following functions: Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for - cases where you cannot use a list. If the *always* keyword argument is - set to ``True`` then the string *s* will always be escaped, even in the - absence of special characters. + cases where you cannot use a list. + + If the *always* keyword argument is set to ``True`` then the string *s* + will always be escaped, even in the absence of special characters. This is + nice for uniformity, for example when escaping a list of strings. + + >>> from shlex import quote + >>> strs = ['escape', 'all of', 'us'] + >>> [quote(s, always=True) for s in strs] + ["'escape'", "'all of'", "'us'"] .. _shlex-quote-warning: From b8ef5ce375a15fce2b37e9ddf9394f66502754d1 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Mon, 3 Jun 2024 23:50:43 +0100 Subject: [PATCH 08/11] Add testQuoteAlways to shlex tests The first assertTrue test checks all strings now start with a single quote, ie they have actually been escaped The second test of assertFalse checks that not all the strings have been escaped when always=False (the default value), since the first, third, and fifth string do not need escaping unless requested. --- Lib/test/test_shlex.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 797c91ee7effdf..4e7454ac67b515 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -337,6 +337,18 @@ def testQuote(self): self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) + def testQuoteAlways(self): + strs = ['hello', 'to the', 'world', 'escape me', 'no-escape-needed'] + + # guarantee escaping all strings + strs_always_escaped = [shlex.quote(s, always=True) for s in strs] + self.assertTrue(all(s.startswith("'") for s in strs_always_escaped)) + + # just escape when necessary ('to the', 'escape me') + strs_necessary_escaped = [shlex.quote(s, always=False) for s in strs] + self.assertFalse(all(s.startswith("'") + for s in strs_necessary_escaped)) + def testJoin(self): for split_command, command in [ (['a ', 'b'], "'a ' b"), From bf7dce0735c62723441041df6ca462deffd9fde8 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Mon, 3 Jun 2024 23:51:09 +0100 Subject: [PATCH 09/11] Add `shlex` subsection to whatsnew for `shlex.quote` additions --- Doc/whatsnew/3.14.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 15216479cc6e5c..94eab329ad25d7 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -183,6 +183,12 @@ ___ Use :func:`pty.openpty` instead. (Contributed by Nikita Sobolev in :gh:`118824`.) +shlex +----- + +* Add keyword-only argument of *always* to :func:`shlex.quote` for forcing + the escaping of the string passed to it. + sqlite3 ------- From ffdc6658874efc4fc47d1720d442c5ad08c4cd7b Mon Sep 17 00:00:00 2001 From: jb2170 Date: Tue, 4 Jun 2024 15:22:49 +0100 Subject: [PATCH 10/11] Explicitly write the expected results of `test_shlex.testQuoteAlways` --- Lib/test/test_shlex.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 4e7454ac67b515..ce06546688a96d 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -341,13 +341,16 @@ def testQuoteAlways(self): strs = ['hello', 'to the', 'world', 'escape me', 'no-escape-needed'] # guarantee escaping all strings - strs_always_escaped = [shlex.quote(s, always=True) for s in strs] - self.assertTrue(all(s.startswith("'") for s in strs_always_escaped)) - - # just escape when necessary ('to the', 'escape me') - strs_necessary_escaped = [shlex.quote(s, always=False) for s in strs] - self.assertFalse(all(s.startswith("'") - for s in strs_necessary_escaped)) + expected = ["'hello'", "'to the'", "'world'", "'escape me'", + "'no-escape-needed'"] + result = [shlex.quote(s, always=True) for s in strs] + self.assertEqual(expected, result) + + # just escape when necessary + expected = ["hello", "'to the'", "world", "'escape me'", + "no-escape-needed"] + result = [shlex.quote(s, always=False) for s in strs] + self.assertEqual(expected, result) def testJoin(self): for split_command, command in [ From 1a7a5cfe4fda37b43bcdb1841f42e888005a1804 Mon Sep 17 00:00:00 2001 From: jb2170 Date: Tue, 10 Sep 2024 22:29:38 +0100 Subject: [PATCH 11/11] Simplify `shlex.quote`'s behaviour to do what it says on the tin --- Doc/library/shlex.rst | 14 +---------- Doc/whatsnew/3.14.rst | 3 +-- Lib/shlex.py | 16 ++++--------- Lib/test/test_shlex.py | 23 +++---------------- ...-05-28-17-24-19.gh-issue-119670.P4J3vX.rst | 3 +-- 5 files changed, 10 insertions(+), 49 deletions(-) diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 408b0cf137755c..a96f0864dc1260 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -49,21 +49,12 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.8 -.. function:: quote(s, *, always=False) +.. function:: quote(s) Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for cases where you cannot use a list. - If the *always* keyword argument is set to ``True`` then the string *s* - will always be escaped, even in the absence of special characters. This is - nice for uniformity, for example when escaping a list of strings. - - >>> from shlex import quote - >>> strs = ['escape', 'all of', 'us'] - >>> [quote(s, always=True) for s in strs] - ["'escape'", "'all of'", "'us'"] - .. _shlex-quote-warning: .. warning:: @@ -107,9 +98,6 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.3 - .. versionchanged:: 3.14 - Added the *always* parameter. - The :mod:`shlex` module defines the following class: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 94eab329ad25d7..3088714c72e566 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -186,8 +186,7 @@ ___ shlex ----- -* Add keyword-only argument of *always* to :func:`shlex.quote` for forcing - the escaping of the string passed to it. +* :func:`shlex.quote` now always quotes its input instead of testing a regex sqlite3 ------- diff --git a/Lib/shlex.py b/Lib/shlex.py index c5c5c0ef420921..280ed75854f92b 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -8,7 +8,6 @@ # changes to tokenize more like Posix shells by Vinay Sajip, July 2016. import os -import re import sys from collections import deque @@ -318,19 +317,12 @@ def join(split_command): return ' '.join(quote(arg) for arg in split_command) -_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search - -def quote(s, *, always=False): +def quote(s): """Return a shell-escaped version of the string *s*.""" - if not s: - return "''" - - if always or _find_unsafe(s) is not None: - # use single quotes, and put single quotes into double quotes - # the string $'b is then quoted as '$'"'"'b' - return "'" + s.replace("'", "'\"'\"'") + "'" + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" - return s def _print_tokens(lexer): while tt := lexer.get_token(): diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index ce06546688a96d..da0fd0e2c70a3b 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -323,12 +323,10 @@ def testUnicodeHandling(self): self.assertEqual(list(s), ref) def testQuote(self): - safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./' unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s unsafe = '"`$\\!' + unicode_sample self.assertEqual(shlex.quote(''), "''") - self.assertEqual(shlex.quote(safeunquoted), safeunquoted) self.assertEqual(shlex.quote('test file name'), "'test file name'") for u in unsafe: self.assertEqual(shlex.quote('test%sname' % u), @@ -337,26 +335,11 @@ def testQuote(self): self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) - def testQuoteAlways(self): - strs = ['hello', 'to the', 'world', 'escape me', 'no-escape-needed'] - - # guarantee escaping all strings - expected = ["'hello'", "'to the'", "'world'", "'escape me'", - "'no-escape-needed'"] - result = [shlex.quote(s, always=True) for s in strs] - self.assertEqual(expected, result) - - # just escape when necessary - expected = ["hello", "'to the'", "world", "'escape me'", - "no-escape-needed"] - result = [shlex.quote(s, always=False) for s in strs] - self.assertEqual(expected, result) - def testJoin(self): for split_command, command in [ - (['a ', 'b'], "'a ' b"), - (['a', ' b'], "a ' b'"), - (['a', ' ', 'b'], "a ' ' b"), + (['a ', 'b'], "'a ' 'b'"), + (['a', ' b'], "'a' ' b'"), + (['a', ' ', 'b'], "'a' ' ' 'b'"), (['"a', 'b"'], '\'"a\' \'b"\''), ]: with self.subTest(command=command): diff --git a/Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst b/Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst index 6fd3dfd2042e42..db033b15f1e06d 100644 --- a/Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst +++ b/Misc/NEWS.d/next/Library/2024-05-28-17-24-19.gh-issue-119670.P4J3vX.rst @@ -1,2 +1 @@ -:func:`shlex.quote` now has an *always* keyword argument for forcing -the escaping of the string passed to it +:func:`shlex.quote` now always quotes its input instead of testing a regex