diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..70971c53b5a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,8 @@ +# When making commits that are strictly formatting/style changes, add the +# commit hash here, so git blame can ignore the change. +# See docs for more details: +# https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +# Example entries: +# # initial black-format +# # rename something internal diff --git a/.gitignore b/.gitignore index 4471ae7e307..1fc0e22a320 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ docs/source/api/generated docs/source/config/options docs/source/config/shortcuts/*.csv docs/source/interactive/magics-generated.txt -docs/source/config/shortcuts/*.csv docs/gh-pages jupyter_notebook/notebook/static/mathjax jupyter_notebook/static/style/*.map diff --git a/.travis.yml b/.travis.yml index f0d4dc6f126..00c5e3f6bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - pip install trio curio --upgrade --upgrade-strategy eager - - pip install 'pytest<4' matplotlib + - pip install pytest 'matplotlib !=3.2.0' mypy - pip install codecov check-manifest --upgrade script: @@ -46,10 +46,11 @@ script: - | if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then # on nightly fake parso known the grammar - cp /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar37.txt /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar38.txt + cp /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar38.txt /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar39.txt fi - cd /tmp && iptest --coverage xml && cd - - pytest IPython + - mypy --ignore-missing-imports -m IPython.terminal.ptutils # On the latest Python (on Linux) only, make sure that the docs build. - | if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then @@ -65,18 +66,27 @@ after_success: matrix: include: - - python: "3.7" + - arch: amd64 + python: "3.7" dist: xenial sudo: true - - python: "3.8-dev" + - arch: amd64 + python: "3.8-dev" dist: xenial sudo: true - - python: "3.7-dev" + - arch: amd64 + python: "3.7-dev" dist: xenial sudo: true - - python: "nightly" + - arch: amd64 + python: "nightly" dist: xenial sudo: true + - arch: arm64 + python: "nightly" + dist: bionic + env: ARM64=True + sudo: true - os: osx language: generic python: 3.6 diff --git a/IPython/__init__.py b/IPython/__init__.py index 4fb77107680..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -65,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. diff --git a/IPython/core/application.py b/IPython/core/application.py index 93639d88e2c..4f679df18e3 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -133,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index 1a7d88991db..fb4cc193250 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -146,20 +146,20 @@ def _should_be_async(cell: str) -> bool: If it works, assume it should be async. Otherwise Return False. - Not handled yet: If the block of code has a return statement as the top + Not handled yet: If the block of code has a return statement as the top level, it will be seen as async. This is a know limitation. """ if sys.version_info > (3, 8): try: code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE - except SyntaxError: + except (SyntaxError, MemoryError): return False try: # we can't limit ourself to ast.parse, as it __accepts__ to parse on # 3.7+, but just does not _compile_ code = compile(cell, "<>", "exec") - except SyntaxError: + except (SyntaxError, MemoryError): try: parse_tree = _async_parse_cell(cell) @@ -167,7 +167,7 @@ def _should_be_async(cell: str) -> bool: v = _AsyncSyntaxErrorVisitor() v.visit(parse_tree) - except SyntaxError: + except (SyntaxError, MemoryError): return False return True return False diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 4906ae9d951..bc114f0f66b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -126,7 +126,7 @@ from contextlib import contextmanager from importlib import import_module -from typing import Iterator, List, Tuple, Iterable, Union +from typing import Iterator, List, Tuple, Iterable from types import SimpleNamespace from traitlets.config.configurable import Configurable @@ -626,6 +626,8 @@ def __init__(self, namespace=None, global_namespace=None, **kwargs): else: self.global_namespace = global_namespace + self.custom_matchers = [] + super(Completer, self).__init__(**kwargs) def complete(self, text, state): @@ -1122,12 +1124,14 @@ def matchers(self): if self.use_jedi: return [ + *self.custom_matchers, self.file_matches, self.magic_matches, self.dict_key_matches, ] else: return [ + *self.custom_matchers, self.python_matches, self.file_matches, self.magic_matches, @@ -1371,18 +1375,18 @@ def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): try_jedi = True try: - # should we check the type of the node is Error ? + # find the first token in the current tree -- if it is a ' or " then we are in a string + completing_string = False try: - # jedi < 0.11 - from jedi.parser.tree import ErrorLeaf - except ImportError: - # jedi >= 0.11 - from parso.tree import ErrorLeaf + first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) + except StopIteration: + pass + else: + # note the value may be ', ", or it may also be ''' or """, or + # in some cases, """what/you/typed..., but all of these are + # strings. + completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} - next_to_last_tree = interpreter._get_module().tree_node.children[-2] - completing_string = False - if isinstance(next_to_last_tree, ErrorLeaf): - completing_string = next_to_last_tree.value.lstrip()[0] in {'"', "'"} # if we are in a string jedi is likely not the right candidate for # now. Skip it. try_jedi = not completing_string @@ -1695,8 +1699,6 @@ def latex_matches(self, text): u"""Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1709,7 +1711,8 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches + if matches: + return s, matches return u'', [] def dispatch_custom_completer(self, text): @@ -1979,8 +1982,8 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # if text is either None or an empty string, rely on the line buffer if (not line_buffer) and full_text: line_buffer = full_text.split('\n')[cursor_line] - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' if self.backslash_combining_completions: # allow deactivation of these on windows. diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 9e592b0817e..7860cb67dcb 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -52,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index 2117edb5c0b..1e0b429d09a 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -29,6 +29,8 @@ from IPython.utils.sysinfo import sys_info from IPython.utils.py3compat import input +from IPython.core.release import __version__ as version + #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- @@ -68,7 +70,7 @@ """ _lite_message_template = """ -If you suspect this is an IPython bug, please report it at: +If you suspect this is an IPython {version} bug, please report it at: https://github.com/ipython/ipython/issues or send an email to the mailing list at {email} @@ -222,5 +224,5 @@ def crash_handler_lite(etype, evalue, tb): else: # we are not in a shell, show generic config config = "c." - print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr) + print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 9ce896ec634..a330baa450e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -153,10 +153,7 @@ def __init__(self, colors=None): # at least raise that limit to 80 chars, which should be enough for # most interactive uses. try: - try: - from reprlib import aRepr # Py 3 - except ImportError: - from repr import aRepr # Py 2 + from reprlib import aRepr aRepr.maxstring = 80 except: # This is only a user-facing convenience, so any error we encounter @@ -283,26 +280,31 @@ def __init__(self, color_scheme=None, completekey=None, # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) self.parser.style = scheme + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - self.stdout.write('\n' + self.shell.get_exception_only()) - - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) - - def new_do_down(self, arg): - OldPdb.do_down(self, arg) - - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + self.stdout.write("\n" + self.shell.get_exception_only()) def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -323,6 +325,8 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: @@ -332,8 +336,21 @@ def print_stack_trace(self, context=None): except (TypeError, ValueError): raise ValueError("Context must be a positive integer") try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass @@ -490,6 +507,16 @@ def print_list_lines(self, filename, first, last): except KeyboardInterrupt: pass + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + def do_list(self, arg): """Print lines of code from the current stack frame """ @@ -625,13 +652,148 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" if arg: - context = int(arg) + try: + context = int(arg) + except ValueError as err: + self.error(err) + return self.print_stack_trace(context) else: self.print_stack_trace() do_w = do_where + def stop_here(self, frame): + hidden = False + if self.skip_hidden: + hidden = frame.f_locals.get("__tracebackhide__", False) + if hidden: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + ## modified version of upstream that skips + # frames with __tracebackide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occured. + self.error("all frames above hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up + +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + def set_trace(frame=None): """ diff --git a/IPython/core/display.py b/IPython/core/display.py index 465c000c55a..424414a662f 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -615,9 +615,12 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): filename = data data = None - self.data = data self.url = url self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data if metadata is not None: self.metadata = metadata @@ -652,23 +655,36 @@ def reload(self): with open(self.filename, self._read_flags) as f: self.data = f.read() elif self.url is not None: - try: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): """Validate that display data is text""" @@ -736,6 +752,11 @@ def _repr_latex_(self): class SVG(DisplayObject): + """Embed an SVG into the display. + + Note if you just want to view a svg image via a URL use `:class:Image` with + a url=URL keyword argument. + """ _read_flags = 'rb' # wrap data in a property, which extracts the tag, discarding @@ -879,7 +900,7 @@ def data(self, data): data = str(data) if isinstance(data, str): - if getattr(self, 'filename', None) is None: + if self.filename is None and self.url is None: warnings.warn("JSON expects JSONable dict or list, not JSON strings") data = json.loads(data) self._data = data @@ -1308,7 +1329,7 @@ def _find_ext(self, s): class Video(DisplayObject): def __init__(self, data=None, url=None, filename=None, embed=False, - mimetype=None, width=None, height=None): + mimetype=None, width=None, height=None, html_attributes="controls"): """Create a video object given raw data or an URL. When this object is returned by an input cell or passed to the @@ -1346,14 +1367,22 @@ def __init__(self, data=None, url=None, filename=None, embed=False, height : int Height in pixels to which to constrain the video in html. If not supplied, defaults to the height of the video. + html_attributes : str + Attributes for the HTML `video element. - """.format(url, width, height) + """.format(url, self.html_attributes, width, height) return output # Embedded videos are base64-encoded. @@ -1411,10 +1441,10 @@ def _repr_html_(self): else: b64_video = b2a_base64(video).decode('ascii').rstrip() - output = """""".format(self.html_attributes, width, height, mimetype, b64_video) return output def reload(self): diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index f651a2a0cf6..1da0458cf08 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -19,7 +19,7 @@ import sys from traitlets.config.configurable import Configurable -from traitlets import List, Dict +from traitlets import List # This used to be defined here - it is imported for backwards compatibility from .display import publish_display_data diff --git a/IPython/core/hooks.py b/IPython/core/hooks.py index b0637d8c154..fa732f7ba82 100644 --- a/IPython/core/hooks.py +++ b/IPython/core/hooks.py @@ -37,7 +37,6 @@ def load_ipython_extension(ip): import os import subprocess -import warnings import sys from .error import TryNext @@ -82,44 +81,6 @@ def editor(self, filename, linenum=None, wait=True): if wait and proc.wait() != 0: raise TryNext() -import tempfile -from ..utils.decorators import undoc - -@undoc -def fix_error_editor(self,filename,linenum,column,msg): - """DEPRECATED - - Open the editor at the given filename, linenumber, column and - show an error message. This is used for correcting syntax errors. - The current implementation only has special support for the VIM editor, - and falls back on the 'editor' hook if VIM is not used. - - Call ip.set_hook('fix_error_editor',yourfunc) to use your own function, - """ - - warnings.warn(""" -`fix_error_editor` is deprecated as of IPython 6.0 and will be removed -in future versions. It appears to be used only for automatically fixing syntax -error that has been broken for a few years and has thus been removed. If you -happened to use this function and still need it please make your voice heard on -the mailing list ipython-dev@python.org , or on the GitHub Issue tracker: -https://github.com/ipython/ipython/issues/9649 """, UserWarning) - - def vim_quickfix_file(): - t = tempfile.NamedTemporaryFile() - t.write('%s:%d:%d:%s\n' % (filename,linenum,column,msg)) - t.flush() - return t - if os.path.basename(self.editor) != 'vim': - self.hooks.editor(filename,linenum) - return - t = vim_quickfix_file() - try: - if os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name): - raise TryNext() - finally: - t.close() - def synchronize_with_editor(self, filename, linenum, column): pass diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index 84aa0a71f0f..e7bc6e7f5a3 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -31,7 +31,6 @@ import tokenize import warnings -from IPython.utils.py3compat import cast_unicode from IPython.core.inputtransformer import (leading_indent, classic_prompt, ipy_prompt, @@ -386,7 +385,7 @@ def check_complete(self, source): finally: self.reset() - def push(self, lines): + def push(self, lines:str) -> bool: """Push one or more lines of input. This stores the given lines and returns a status code indicating @@ -408,6 +407,7 @@ def push(self, lines): this value is also stored as a private attribute (``_is_complete``), so it can be queried at any time. """ + assert isinstance(lines, str) self._store(lines) source = self.source @@ -677,7 +677,7 @@ def transform_cell(self, cell): finally: self.reset() - def push(self, lines): + def push(self, lines:str) -> bool: """Push one or more lines of IPython input. This stores the given lines and returns a status code indicating @@ -700,9 +700,8 @@ def push(self, lines): this value is also stored as a private attribute (_is_complete), so it can be queried at any time. """ - + assert isinstance(lines, str) # We must ensure all input is pure unicode - lines = cast_unicode(lines, self.encoding) # ''.splitlines() --> [], but we need to push the empty line to transformers lines_list = lines.splitlines() if not lines_list: diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index 1c35eb64f32..afeca93cc0e 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -278,8 +278,8 @@ def escaped_commands(line): _initial_space_re = re.compile(r'\s*') _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 04f423274da..0443e6829b4 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -405,8 +405,8 @@ def transform(self, lines): return lines_before + [new_line] + lines_after _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, @@ -674,8 +674,8 @@ def check_complete(self, cell: str): while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types: tokens_by_line[-1].pop() - if len(tokens_by_line) == 1 and not tokens_by_line[-1]: - return 'incomplete', 0 + if not tokens_by_line[-1]: + return 'incomplete', find_last_indent(lines) if tokens_by_line[-1][-1].string == ':': # The last line starts a block (e.g. 'if foo:') diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index dc8bd236d3b..ddb1b64ea78 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -13,7 +13,6 @@ import abc import ast -import asyncio import atexit import builtins as builtin_mod import functools @@ -87,6 +86,7 @@ # NoOpContext is deprecated, but ipykernel imports it from here. # See https://github.com/ipython/ipykernel/issues/157 +# (2016, let's try to remove than in IPython 8.0) from IPython.utils.contexts import NoOpContext try: @@ -695,6 +695,13 @@ def __init__(self, ipython_dir=None, profile_dir=None, self.events.trigger('shell_initialized', self) atexit.register(self.atexit_operations) + # The trio runner is used for running Trio in the foreground thread. It + # is different from `_trio_runner(async_fn)` in `async_helpers.py` + # which calls `trio.run()` for every cell. This runner runs all cells + # inside a single Trio event loop. If used, it is set from + # `ipykernel.kernelapp`. + self.trio_runner = None + def get_ipython(self): """Return the currently running IPython instance.""" return self @@ -715,6 +722,9 @@ def set_autoindent(self,value=None): else: self.autoindent = value + def set_trio_runner(self, tr): + self.trio_runner = tr + #------------------------------------------------------------------------- # init_* methods called by __init__ #------------------------------------------------------------------------- @@ -2201,14 +2211,23 @@ def complete(self, text, line=None, cursor_pos=None): with self.builtin_trap: return self.Completer.complete(text, line, cursor_pos) - def set_custom_completer(self, completer, pos=0): + def set_custom_completer(self, completer, pos=0) -> None: """Adds a new custom completer function. The position argument (defaults to 0) is the index in the completers - list where you want the completer to be inserted.""" + list where you want the completer to be inserted. + + `completer` should have the following signature:: + + def completion(self: Completer, text: string) -> List[str]: + raise NotImplementedError + + It will be bound to the current Completer instance and pass some text + and return a list with current completions to suggest to the user. + """ - newcomp = types.MethodType(completer,self.Completer) - self.Completer.matchers.insert(pos,newcomp) + newcomp = types.MethodType(completer, self.Completer) + self.Completer.custom_matchers.insert(pos,newcomp) def set_completer_frame(self, frame=None): """Set the frame of the completer.""" @@ -2865,7 +2884,9 @@ def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures # when this is the case, we want to run it using the pseudo_sync_runner # so that code can invoke eventloops (for example via the %run , and # `%paste` magic. - if self.should_run_async(raw_cell): + if self.trio_runner: + runner = self.trio_runner + elif self.should_run_async(raw_cell): runner = self.loop_runner else: runner = _pseudo_sync_runner @@ -3298,6 +3319,9 @@ async def run_code(self, code_obj, result=None, *, async_=False): False : successful execution. True : an error occurred. """ + # special value to say that anything above is IPython and should be + # hidden. + __tracebackhide__ = "__ipython_bottom__" # Set our own excepthook in case the user code tries to call it # directly, so that the IPython crash handler doesn't get triggered old_excepthook, sys.excepthook = sys.excepthook, self.excepthook diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 906d3d5ebdf..a8feb755386 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -5,7 +5,6 @@ from logging import error import io from pprint import pformat -import textwrap import sys from warnings import warn @@ -365,13 +364,25 @@ def xmode(self, parameter_s=''): Valid modes: Plain, Context, Verbose, and Minimal. - If called without arguments, acts as a toggle.""" + If called without arguments, acts as a toggle. + + When in verbose mode the value --show (and --hide) + will respectively show (or hide) frames with ``__tracebackhide__ = + True`` value set. + """ def xmode_switch_err(name): warn('Error changing %s exception modes.\n%s' % (name,sys.exc_info()[1])) shell = self.shell + if parameter_s.strip() == "--show": + shell.InteractiveTB.skip_hidden = False + return + if parameter_s.strip() == "--hide": + shell.InteractiveTB.skip_hidden = True + return + new_mode = parameter_s.strip().capitalize() try: shell.InteractiveTB.set_mode(mode=new_mode) diff --git a/IPython/core/magics/code.py b/IPython/core/magics/code.py index 41aa37ca7cf..a1841384651 100644 --- a/IPython/core/magics/code.py +++ b/IPython/core/magics/code.py @@ -29,7 +29,6 @@ from IPython.core.magic import Magics, magics_class, line_magic from IPython.core.oinspect import find_file, find_source_lines from IPython.testing.skipdoctest import skip_doctest -from IPython.utils import py3compat from IPython.utils.contexts import preserve_keys from IPython.utils.path import get_py_filename from warnings import warn @@ -214,9 +213,9 @@ def save(self, parameter_s=''): force = 'f' in opts append = 'a' in opts mode = 'a' if append else 'w' - ext = u'.ipy' if raw else u'.py' + ext = '.ipy' if raw else '.py' fname, codefrom = args[0], " ".join(args[1:]) - if not fname.endswith((u'.py',u'.ipy')): + if not fname.endswith(('.py','.ipy')): fname += ext file_exists = os.path.isfile(fname) if file_exists and not force and not append: @@ -233,14 +232,13 @@ def save(self, parameter_s=''): except (TypeError, ValueError) as e: print(e.args[0]) return - out = py3compat.cast_unicode(cmds) with io.open(fname, mode, encoding="utf-8") as f: if not file_exists or not append: - f.write(u"# coding: utf-8\n") - f.write(out) + f.write("# coding: utf-8\n") + f.write(cmds) # make sure we end on a newline - if not out.endswith(u'\n'): - f.write(u'\n') + if not cmds.endswith('\n'): + f.write('\n') print('The following commands were written to file `%s`:' % fname) print(cmds) diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index 438d0b5f0ad..dc6cdf00e29 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -688,17 +688,16 @@ def run(self, parameter_s='', runner=None, modulename = opts["m"][0] modpath = find_mod(modulename) if modpath is None: - warn('%r is not a valid modulename on sys.path'%modulename) - return + msg = '%r is not a valid modulename on sys.path'%modulename + raise Exception(msg) arg_lst = [modpath] + arg_lst try: fpath = None # initialize to make sure fpath is in scope later fpath = arg_lst[0] filename = file_finder(fpath) except IndexError: - warn('you must provide at least a filename.') - print('\n%run:\n', oinspect.getdoc(self.run)) - return + msg = 'you must provide at least a filename.' + raise Exception(msg) except IOError as e: try: msg = str(e) @@ -706,13 +705,17 @@ def run(self, parameter_s='', runner=None, msg = e.message if os.name == 'nt' and re.match(r"^'.*'$",fpath): warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"') - error(msg) - return + raise Exception(msg) + except TypeError: + if fpath in sys.meta_path: + filename = "" + else: + raise if filename.lower().endswith(('.ipy', '.ipynb')): with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = filename - self.shell.safe_execfile_ipy(filename) + self.shell.safe_execfile_ipy(filename, raise_exceptions=True) return # Control the response to exit() calls made by the script being run diff --git a/IPython/core/magics/namespace.py b/IPython/core/magics/namespace.py index cef6ddba8d7..acc4620549b 100644 --- a/IPython/core/magics/namespace.py +++ b/IPython/core/magics/namespace.py @@ -208,12 +208,6 @@ def psearch(self, parameter_s=''): %psearch -l list all available object types """ - try: - parameter_s.encode('ascii') - except UnicodeEncodeError: - print('Python identifiers can only contain ascii characters.') - return - # default namespaces to be searched def_search = ['user_local', 'user_global', 'builtin'] diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index ddc9b6f5874..90da7e22803 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -25,6 +25,7 @@ from IPython.utils.process import abbrev_cwd from IPython.utils.terminal import set_term_title from traitlets import Bool +from warnings import warn @magics_class @@ -48,8 +49,15 @@ def __init__(self, shell=None, **kwargs): winext = os.environ['pathext'].replace(';','|').replace('.','') except KeyError: winext = 'exe|com|bat|py' - - self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + try: + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + except re.error: + warn("Seems like your pathext environmental " + "variable is malformed. Please check it to " + "enable a proper handle of file extensions " + "managed for your system") + winext = 'exe|com|bat|py' + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) # call up the chain super().__init__(shell=shell, **kwargs) diff --git a/IPython/core/magics/packaging.py b/IPython/core/magics/packaging.py index 6477c7defc7..cfee7865f5d 100644 --- a/IPython/core/magics/packaging.py +++ b/IPython/core/magics/packaging.py @@ -12,7 +12,6 @@ import re import shlex import sys -from subprocess import Popen, PIPE from IPython.core.magic import Magics, magics_class, line_magic @@ -101,4 +100,4 @@ def conda(self, line): extra_args.extend(["--prefix", sys.prefix]) self.shell.system(' '.join([conda, command] + extra_args + args)) - print("\nNote: you may need to restart the kernel to use updated packages.") \ No newline at end of file + print("\nNote: you may need to restart the kernel to use updated packages.") diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 0ad709405de..ab25eeeffca 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -22,7 +22,8 @@ from textwrap import dedent import types import io as stdlib_io -from itertools import zip_longest + +from typing import Union # IPython's own from IPython.core import page @@ -82,7 +83,7 @@ def pylight(code): def object_info(**kw): """Make an object info dict with all fields present.""" - infodict = dict(zip_longest(info_fields, [None])) + infodict = {k:None for k in info_fields} infodict.update(kw) return infodict @@ -110,7 +111,7 @@ def get_encoding(obj): encoding, lines = openpy.detect_encoding(buffer.readline) return encoding -def getdoc(obj): +def getdoc(obj) -> Union[str,None]: """Stable wrapper around inspect.getdoc. This can't crash because of attribute problems. @@ -128,11 +129,10 @@ def getdoc(obj): if isinstance(ds, str): return inspect.cleandoc(ds) docstr = inspect.getdoc(obj) - encoding = get_encoding(obj) - return py3compat.cast_unicode(docstr, encoding=encoding) + return docstr -def getsource(obj, oname=''): +def getsource(obj, oname='') -> Union[str,None]: """Wrapper around inspect.getsource. This can be modified by other projects to provide customized source @@ -158,18 +158,15 @@ def getsource(obj, oname=''): if fn is not None: encoding = get_encoding(fn) oname_prefix = ('%s.' % oname) if oname else '' - sources.append(cast_unicode( - ''.join(('# ', oname_prefix, attrname)), - encoding=encoding)) + sources.append(''.join(('# ', oname_prefix, attrname))) if inspect.isfunction(fn): sources.append(dedent(getsource(fn))) else: # Default str/repr only prints function name, # pretty.pretty prints module name too. - sources.append(cast_unicode( - '%s%s = %s\n' % ( - oname_prefix, attrname, pretty(fn)), - encoding=encoding)) + sources.append( + '%s%s = %s\n' % (oname_prefix, attrname, pretty(fn)) + ) if sources: return '\n'.join(sources) else: @@ -191,8 +188,7 @@ def getsource(obj, oname=''): except TypeError: return None - encoding = get_encoding(obj) - return cast_unicode(src, encoding=encoding) + return src def is_simple_callable(obj): @@ -202,8 +198,7 @@ def is_simple_callable(obj): @undoc def getargspec(obj): - """Wrapper around :func:`inspect.getfullargspec` on Python 3, and - :func:inspect.getargspec` on Python 2. + """Wrapper around :func:`inspect.getfullargspec` In addition to functions and methods, this can also handle objects with a ``__call__`` attribute. @@ -289,7 +284,7 @@ def _get_wrapped(obj): return orig_obj return obj -def find_file(obj): +def find_file(obj) -> str: """Find the absolute path to the file where an object was defined. This is essentially a robust wrapper around `inspect.getabsfile`. @@ -370,18 +365,17 @@ def __init__(self, color_table=InspectColors, self.str_detail_level = str_detail_level self.set_active_scheme(scheme) - def _getdef(self,obj,oname=''): + def _getdef(self,obj,oname='') -> Union[str,None]: """Return the call signature for any callable object. If any exception is generated, None is returned instead and the exception is suppressed.""" try: - hdef = _render_signature(signature(obj), oname) - return cast_unicode(hdef) + return _render_signature(signature(obj), oname) except: return None - def __head(self,h): + def __head(self,h) -> str: """Return a header string with proper colors.""" return '%s%s%s' % (self.color_table.active_colors.header,h, self.color_table.active_colors.normal) @@ -517,29 +511,8 @@ def pfile(self, obj, oname=''): # 0-offset, so we must adjust. page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1) - def _format_fields(self, fields, title_width=0): - """Formats a list of fields for display. - - Parameters - ---------- - fields : list - A list of 2-tuples: (field_title, field_content) - title_width : int - How many characters to pad titles to. Default to longest title. - """ - out = [] - header = self.__head - if title_width == 0: - title_width = max(len(title) + 2 for title, _ in fields) - for title, content in fields: - if len(content.splitlines()) > 1: - title = header(title + ':') + '\n' - else: - title = header((title + ':').ljust(title_width)) - out.append(cast_unicode(title) + cast_unicode(content)) - return "\n".join(out) - def _mime_format(self, text, formatter=None): + def _mime_format(self, text:str, formatter=None) -> dict: """Return a mime bundle representation of the input text. - if `formatter` is None, the returned mime bundle has @@ -555,7 +528,6 @@ def _mime_format(self, text, formatter=None): Formatters returning strings are supported but this behavior is deprecated. """ - text = cast_unicode(text) defaults = { 'text/plain': text, 'text/html': '
' + text + '
' @@ -618,7 +590,7 @@ def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0): 'text/html': '', } - def append_field(bundle, title, key, formatter=None): + def append_field(bundle, title:str, key:str, formatter=None): field = info[key] if field is not None: formatted_field = self._mime_format(field, formatter) @@ -738,7 +710,8 @@ def _info(self, obj, oname='', info=None, detail_level=0) -> dict: Returns ======= - An object info dict with known fields from `info_fields`. + An object info dict with known fields from `info_fields`. Keys are + strings, values are string or None. """ if info is None: @@ -1015,7 +988,7 @@ def psearch(self,pattern,ns_table,ns_search=[], page.page('\n'.join(sorted(search_result))) -def _render_signature(obj_signature, obj_name): +def _render_signature(obj_signature, obj_name) -> str: """ This was mostly taken from inspect.Signature.__str__. Look there for the comments. diff --git a/IPython/core/page.py b/IPython/core/page.py index afdded29b1a..ed16b617812 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -341,32 +341,3 @@ def page_more(): return False else: return True - - -def snip_print(str,width = 75,print_full = 0,header = ''): - """Print a string snipping the midsection to fit in width. - - print_full: mode control: - - - 0: only snip long strings - - 1: send to page() directly. - - 2: snip long strings and ask for full length viewing with page() - - Return 1 if snipping was necessary, 0 otherwise.""" - - if print_full == 1: - page(header+str) - return 0 - - print(header, end=' ') - if len(str) < width: - print(str) - snip = 0 - else: - whalf = int((width -5)/2) - print(str[:whalf] + ' <...> ' + str[-whalf:]) - snip = 1 - if snip and print_full == 2: - if py3compat.input(header+' Snipped. View (y/n)? [N]').lower() == 'y': - page(str) - return snip diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index dbf185e6a42..bf801f999c4 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -37,7 +37,7 @@ class PrefilterError(Exception): # RegExp to identify potential function names -re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$') +re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$') # RegExp to exclude strings with this start from autocalling. In # particular, all binary operators should be excluded, so that if foo is diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 97434e3d0b5..9a1bae55ac5 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -181,9 +181,10 @@ def list_profile_dirs(self): profiles = list_profiles_in(os.getcwd()) if profiles: print() - print("Available profiles in current directory (%s):" % os.getcwd()) - self._print_profiles(profiles) - + print( + "Profiles from CWD have been removed for security reason, see CVE-2022-21699:" + ) + print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=") diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 3199dfd5d64..2c48e4c2f1c 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -186,7 +186,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: - 1. ``os.getcwd()`` + 1. ``os.getcwd()`` # removed for security reason. 2. ``ipython_dir`` Parameters @@ -198,7 +198,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): will be "profile_". """ dirname = u'profile_' + name - paths = [os.getcwd(), ipython_dir] + paths = [ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): diff --git a/IPython/core/pylabtools.py b/IPython/core/pylabtools.py index d42c088980c..cb1ce811984 100644 --- a/IPython/core/pylabtools.py +++ b/IPython/core/pylabtools.py @@ -123,14 +123,18 @@ def print_figure(fig, fmt='png', bbox_inches='tight', **kwargs): } # **kwargs get higher priority kw.update(kwargs) - + bytes_io = BytesIO() + if fig.canvas is None: + from matplotlib.backend_bases import FigureCanvasBase + FigureCanvasBase(fig) + fig.canvas.print_figure(bytes_io, **kw) data = bytes_io.getvalue() if fmt == 'svg': data = data.decode('utf-8') return data - + def retina_figure(fig, **kwargs): """format a figure as a pixel-doubled (retina) PNG""" pngdata = print_figure(fig, fmt='retina', **kwargs) diff --git a/IPython/core/release.py b/IPython/core/release.py index a928a6b8825..a6f3cf81f7d 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 10 -_version_patch = 1 +_version_minor = 16 +_version_patch = 3 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index a09192f255c..9e8bfbfbb81 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -60,6 +60,10 @@ colours.""", "Disable using colors for info related things." ) +addflag('ignore-cwd', 'InteractiveShellApp.ignore_cwd', + "Exclude the current working directory from sys.path", + "Include the current working directory in sys.path", +) nosep_config = Config() nosep_config.InteractiveShell.separate_in = '' nosep_config.InteractiveShell.separate_out = '' @@ -168,6 +172,12 @@ class InteractiveShellApp(Configurable): When False, pylab mode should not import any names into the user namespace. """ ).tag(config=True) + ignore_cwd = Bool( + False, + help="""If True, IPython will not add the current working directory to sys.path. + When False, the current working directory is added to sys.path, allowing imports + of modules defined in the current directory.""" + ).tag(config=True) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) # whether interact-loop should start @@ -189,8 +199,10 @@ def init_path(self): .. versionchanged:: 7.2 Try to insert after the standard library, instead of first. + .. versionchanged:: 8.0 + Allow optionally not including the current directory in sys.path """ - if '' in sys.path: + if '' in sys.path or self.ignore_cwd: return for idx, path in enumerate(sys.path): parent, last_part = os.path.split(path) @@ -404,6 +416,10 @@ def _run_cmd_line_code(self): fname = self.file_to_run if os.path.isdir(fname): fname = os.path.join(fname, "__main__.py") + if not os.path.exists(fname): + self.log.warning("File '%s' doesn't exist", fname) + if not self.interact: + self.exit(2) try: self._exec_file(fname, shell_futures=True) except: diff --git a/IPython/core/tests/refbug.py b/IPython/core/tests/refbug.py index b8de4c81078..92e2eead347 100644 --- a/IPython/core/tests/refbug.py +++ b/IPython/core/tests/refbug.py @@ -16,7 +16,6 @@ #----------------------------------------------------------------------------- # Module imports #----------------------------------------------------------------------------- -import sys from IPython import get_ipython diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index 2432d8b5e4e..11c475874d7 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -3,13 +3,12 @@ Should only trigger on python 3.5+ or will have syntax errors. """ -import sys from itertools import chain, repeat import nose.tools as nt from textwrap import dedent, indent from unittest import TestCase from IPython.testing.decorators import skip_without - +import sys iprc = lambda x: ip.run_cell(dedent(x)).raise_error() iprc_nr = lambda x: ip.run_cell(dedent(x)) @@ -276,6 +275,13 @@ def test_autoawait(self): await sleep(0.1) """ ) + + if sys.version_info < (3,9): + # new pgen parser in 3.9 does not raise MemoryError on too many nested + # parens anymore + def test_memory_error(self): + with self.assertRaises(MemoryError): + iprc("(" * 200 + ")" * 200) @skip_without('curio') def test_autoawait_curio(self): diff --git a/IPython/core/tests/test_autocall.py b/IPython/core/tests/test_autocall.py index 10a4e0d477d..ded9f78858a 100644 --- a/IPython/core/tests/test_autocall.py +++ b/IPython/core/tests/test_autocall.py @@ -7,7 +7,6 @@ """ from IPython.core.splitinput import LineInfo from IPython.core.prefilter import AutocallChecker -from IPython.utils import py3compat def doctest_autocall(): """ diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 2920d453936..2c19e2e0187 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -224,20 +224,27 @@ def test_latex_completions(self): nt.assert_in("\\alpha", matches) nt.assert_in("\\aleph", matches) + def test_latex_no_results(self): + """ + forward latex should really return nothing in either field if nothing is found. + """ + ip = get_ipython() + text, matches = ip.Completer.latex_matches("\\really_i_should_match_nothing") + nt.assert_equal(text, "") + nt.assert_equal(matches, []) + def test_back_latex_completion(self): ip = get_ipython() # do not return more than 1 matches fro \beta, only the latex one. name, matches = ip.complete("\\β") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\beta") + nt.assert_equal(matches, ['\\beta']) def test_back_unicode_completion(self): ip = get_ipython() name, matches = ip.complete("\\Ⅴ") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\ROMAN NUMERAL FIVE") + nt.assert_equal(matches, ["\\ROMAN NUMERAL FIVE"]) def test_forward_unicode_completion(self): ip = get_ipython() diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index dcfd9a42438..9fdc944e4d0 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -4,12 +4,24 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import bdb +import builtins +import os +import signal +import subprocess import sys +import time import warnings +from subprocess import PIPE, CalledProcessError, check_output +from tempfile import NamedTemporaryFile +from textwrap import dedent +from unittest.mock import patch import nose.tools as nt from IPython.core import debugger +from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE +from IPython.testing.decorators import skip_win32 #----------------------------------------------------------------------------- # Helper classes, from CPython's Pdb test suite @@ -223,3 +235,92 @@ def can_exit(): >>> sys.settrace(old_trace) ''' + + +def test_interruptible_core_debugger(): + """The debugger can be interrupted. + + The presumption is there is some mechanism that causes a KeyboardInterrupt + (this is implemented in ipykernel). We want to ensure the + KeyboardInterrupt cause debugging to cease. + """ + def raising_input(msg="", called=[0]): + called[0] += 1 + if called[0] == 1: + raise KeyboardInterrupt() + else: + raise AssertionError("input() should only be called once!") + + with patch.object(builtins, "input", raising_input): + debugger.InterruptiblePdb().set_trace() + # The way this test will fail is by set_trace() never exiting, + # resulting in a timeout by the test runner. The alternative + # implementation would involve a subprocess, but that adds issues with + # interrupting subprocesses that are rather complex, so it's simpler + # just to do it this way. + +@skip_win32 +def test_xmode_skip(): + """that xmode skip frames + + Not as a doctest as pytest does not run doctests. + """ + import pexpect + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + child = pexpect.spawn( + sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env + ) + child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE + + child.expect("IPython") + child.expect("\n") + child.expect_exact("In [1]") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + raise ValueError + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + child.expect_exact("skipping") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + from IPython.core.debugger import set_trace + set_trace() + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + + child.expect("ipdb>") + child.sendline("w") + child.expect("hidden") + child.expect("ipdb>") + child.sendline("skip_hidden false") + child.sendline("w") + child.expect("__traceba") + child.expect("ipdb>") + + child.close() diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 1fed51127a1..95f1eb622e4 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -14,7 +14,7 @@ from IPython.utils.io import capture_output from IPython.utils.tempdir import NamedFileInTemporaryDirectory from IPython import paths as ipath -from IPython.testing.tools import AssertPrints, AssertNotPrints +from IPython.testing.tools import AssertNotPrints import IPython.testing.decorators as dec @@ -72,6 +72,40 @@ def test_retina_png(): nt.assert_equal(md['width'], 1) nt.assert_equal(md['height'], 1) +def test_embed_svg_url(): + import gzip + from io import BytesIO + svg_data = b'' + url = 'http://test.com/circle.svg' + + gzip_svg = BytesIO() + with gzip.open(gzip_svg, 'wb') as fp: + fp.write(svg_data) + gzip_svg = gzip_svg.getvalue() + + def mocked_urlopen(*args, **kwargs): + class MockResponse: + def __init__(self, svg): + self._svg_data = svg + self.headers = {'content-type': 'image/svg+xml'} + + def read(self): + return self._svg_data + + if args[0] == url: + return MockResponse(svg_data) + elif args[0] == url + 'z': + ret= MockResponse(gzip_svg) + ret.headers['content-encoding']= 'gzip' + return ret + return MockResponse(None) + + with mock.patch('urllib.request.urlopen', side_effect=mocked_urlopen): + svg = display.SVG(url=url) + nt.assert_true(svg._repr_svg_().startswith('=3.9 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, +) -> bool\ +''', # Python >=3.7 '''\ long_function( diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index 0e61b4693f7..ca447b3d0b7 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -115,3 +115,13 @@ def __call__(self, x): finally: del ip.user_ns['x'] ip.magic('autocall 0') + + +def test_autocall_should_support_unicode(): + ip.magic('autocall 2') + ip.user_ns['π'] = lambda x: x + try: + nt.assert_equal(ip.prefilter('π 3'),'π(3)') + finally: + ip.magic('autocall 0') + del ip.user_ns['π'] diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index 181e99f9b84..7b64aab111a 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -61,7 +61,7 @@ def test_figure_to_jpeg(): ax = fig.add_subplot(1,1,1) ax.plot([1,2,3]) plt.draw() - jpeg = pt.print_figure(fig, 'jpeg', quality=50)[:100].lower() + jpeg = pt.print_figure(fig, 'jpeg', pil_kwargs={'optimize': 50})[:100].lower() assert jpeg.startswith(_JPEG) def test_retina_figure(): @@ -248,3 +248,9 @@ def test_qt_gtk(self): def test_no_gui_backends(): for k in ['agg', 'svg', 'pdf', 'ps']: assert k not in pt.backend2gui + + +def test_figure_no_canvas(): + fig = Figure() + fig.canvas = None + pt.print_figure(fig) diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 38d71b31740..eff832b3fc0 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -402,6 +402,25 @@ def test_run_nb(self): nt.assert_equal(_ip.user_ns['answer'], 42) + def test_run_nb_error(self): + """Test %run notebook.ipynb error""" + from nbformat import v4, writes + # %run when a file name isn't provided + nt.assert_raises(Exception, _ip.magic, "run") + + # %run when a file doesn't exist + nt.assert_raises(Exception, _ip.magic, "run foobar.ipynb") + + # %run on a notebook with an error + nb = v4.new_notebook( + cells=[ + v4.new_code_cell("0/0") + ] + ) + src = writes(nb, version=4) + self.mktmp(src, ext='.ipynb') + nt.assert_raises(Exception, _ip.magic, "run %s" % self.fname) + def test_file_options(self): src = ('import sys\n' 'a = " ".join(sys.argv[1:])\n') diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index e77f44416c3..3751117b692 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -252,6 +252,17 @@ def test_non_syntaxerror(self): with tt.AssertPrints('QWERTY'): ip.showsyntaxerror() +import sys +if sys.version_info < (3,9): + """ + New 3.9 Pgen Parser does not raise Memory error, except on failed malloc. + """ + class MemoryErrorTest(unittest.TestCase): + def test_memoryerror(self): + memoryerror_code = "(" * 200 + ")" * 200 + with tt.AssertPrints("MemoryError"): + ip.run_cell(memoryerror_code) + class Python3ChainedExceptionsTest(unittest.TestCase): DIRECT_CAUSE_ERROR_CODE = """ @@ -296,6 +307,25 @@ def test_suppress_exception_chaining(self): tt.AssertPrints("ValueError", suppress=False): ip.run_cell(self.SUPPRESS_CHAINING_CODE) + def test_plain_direct_cause_error(self): + with tt.AssertPrints(["KeyError", "NameError", "direct cause"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE) + ip.run_cell("%xmode Verbose") + + def test_plain_exception_during_handling_error(self): + with tt.AssertPrints(["KeyError", "NameError", "During handling"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) + ip.run_cell("%xmode Verbose") + + def test_plain_suppress_exception_chaining(self): + with tt.AssertNotPrints("ZeroDivisionError"), \ + tt.AssertPrints("ValueError", suppress=False): + ip.run_cell("%xmode Plain") + ip.run_cell(self.SUPPRESS_CHAINING_CODE) + ip.run_cell("%xmode Verbose") + class RecursionTest(unittest.TestCase): DEFINITIONS = """ diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 0864ba456ef..45e22bd7b94 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -101,10 +101,7 @@ import tokenize import traceback -try: # Python 2 - generate_tokens = tokenize.generate_tokens -except AttributeError: # Python 3 - generate_tokens = tokenize.tokenize +from tokenize import generate_tokens # For purposes of monkeypatching inspect to fix a bug in it. from inspect import getsourcefile, getfile, getmodule, \ @@ -530,6 +527,30 @@ def _set_ostream(self, val): ostream = property(_get_ostream, _set_ostream) + def get_parts_of_chained_exception(self, evalue): + def get_chained_exception(exception_value): + cause = getattr(exception_value, '__cause__', None) + if cause: + return cause + if getattr(exception_value, '__suppress_context__', False): + return None + return getattr(exception_value, '__context__', None) + + chained_evalue = get_chained_exception(evalue) + + if chained_evalue: + return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ + + def prepare_chained_exception_message(self, cause): + direct_cause = "\nThe above exception was the direct cause of the following exception:\n" + exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" + + if cause: + message = [[direct_cause]] + else: + message = [[exception_during_handling]] + return message + def set_colors(self, *args, **kw): """Shorthand access to the color table scheme selector method.""" @@ -603,7 +624,13 @@ def __call__(self, etype, value, elist): self.ostream.write(self.text(etype, value, elist)) self.ostream.write('\n') - def structured_traceback(self, etype, value, elist, tb_offset=None, + def _extract_tb(self, tb): + if tb: + return traceback.extract_tb(tb) + else: + return None + + def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, context=5): """Return a color formatted string with the traceback info. @@ -612,15 +639,16 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, etype : exception type Type of the exception raised. - value : object + evalue : object Data stored in the exception - elist : list - List of frames, see class docstring for details. + etb : object + If list: List of frames, see class docstring for details. + If Traceback: Traceback of the exception. tb_offset : int, optional Number of frames in the traceback to skip. If not given, the - instance value is used (set in constructor). + instance evalue is used (set in constructor). context : int, optional Number of lines of context information to print. @@ -629,6 +657,19 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, ------- String with formatted exception. """ + # This is a workaround to get chained_exc_ids in recursive calls + # etb should not be a tuple if structured_traceback is not recursive + if isinstance(etb, tuple): + etb, chained_exc_ids = etb + else: + chained_exc_ids = set() + + if isinstance(etb, list): + elist = etb + elif etb is not None: + elist = self._extract_tb(etb) + else: + elist = [] tb_offset = self.tb_offset if tb_offset is None else tb_offset Colors = self.Colors out_list = [] @@ -641,9 +682,25 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, (Colors.normalEm, Colors.Normal) + '\n') out_list.extend(self._format_list(elist)) # The exception info should be a single entry in the list. - lines = ''.join(self._format_exception_only(etype, value)) + lines = ''.join(self._format_exception_only(etype, evalue)) out_list.append(lines) + exception = self.get_parts_of_chained_exception(evalue) + + if exception and not id(exception[1]) in chained_exc_ids: + chained_exception_message = self.prepare_chained_exception_message( + evalue.__cause__)[0] + etype, evalue, etb = exception + # Trace exception to avoid infinite 'cause' loop + chained_exc_ids.add(id(exception[1])) + chained_exceptions_tb_offset = 0 + out_list = ( + self.structured_traceback( + etype, evalue, (etb, chained_exc_ids), + chained_exceptions_tb_offset, context) + + chained_exception_message + + out_list) + return out_list def _format_list(self, extracted_list): @@ -763,7 +820,7 @@ def get_exception_only(self, etype, value): etype : exception type value : exception value """ - return ListTB.structured_traceback(self, etype, value, []) + return ListTB.structured_traceback(self, etype, value) def show_exception_only(self, etype, evalue): """Only print the exception type and message, without a traceback. @@ -822,13 +879,36 @@ def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, self.check_cache = check_cache self.debugger_cls = debugger_cls or debugger.Pdb + self.skip_hidden = True def format_records(self, records, last_unique, recursion_repeat): """Format the stack frames of the traceback""" frames = [] + + skipped = 0 for r in records[:last_unique+recursion_repeat+1]: - #print '*** record:',file,lnum,func,lines,index # dbg + if self.skip_hidden: + if r[0].f_locals.get("__tracebackhide__", 0): + skipped += 1 + continue + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) + skipped = 0 + frames.append(self.format_record(*r)) + + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) if recursion_repeat: frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat) @@ -1013,16 +1093,6 @@ def linereader(file=file, lnum=[lnum], getline=linecache.getline): _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format))) - def prepare_chained_exception_message(self, cause): - direct_cause = "\nThe above exception was the direct cause of the following exception:\n" - exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" - - if cause: - message = [[direct_cause]] - else: - message = [[exception_during_handling]] - return message - def prepare_header(self, etype, long_version=False): colors = self.Colors # just a shorthand + quicker name lookup colorsnormal = colors.Normal # used a lot @@ -1076,8 +1146,6 @@ def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_con head = self.prepare_header(etype, self.long_header) records = self.get_records(etb, number_of_lines_of_context, tb_offset) - if records is None: - return "" last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records) @@ -1117,20 +1185,6 @@ def get_records(self, etb, number_of_lines_of_context, tb_offset): info('\nUnfortunately, your original traceback can not be constructed.\n') return None - def get_parts_of_chained_exception(self, evalue): - def get_chained_exception(exception_value): - cause = getattr(exception_value, '__cause__', None) - if cause: - return cause - if getattr(exception_value, '__suppress_context__', False): - return None - return getattr(exception_value, '__context__', None) - - chained_evalue = get_chained_exception(evalue) - - if chained_evalue: - return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ - def structured_traceback(self, etype, evalue, etb, tb_offset=None, number_of_lines_of_context=5): """Return a nice text document describing the traceback.""" @@ -1273,12 +1327,6 @@ def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, # set_mode also sets the tb_join_char attribute self.set_mode(mode) - def _extract_tb(self, tb): - if tb: - return traceback.extract_tb(tb) - else: - return None - def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5): tb_offset = self.tb_offset if tb_offset is None else tb_offset mode = self.mode @@ -1294,9 +1342,8 @@ def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines # out-of-date source code. self.check_cache() # Now we can extract and format the exception - elist = self._extract_tb(tb) return ListTB.structured_traceback( - self, etype, value, elist, tb_offset, number_of_lines_of_context + self, etype, value, tb, tb_offset, number_of_lines_of_context ) def stb2text(self, stb): @@ -1380,7 +1427,11 @@ def structured_traceback(self, etype=None, value=None, tb=None, tb_offset=None, number_of_lines_of_context=5): if etype is None: etype, value, tb = sys.exc_info() - self.tb = tb + if isinstance(tb, tuple): + # tb is a tuple if this is a chained exception. + self.tb = tb[0] + else: + self.tb = tb return FormattedTB.structured_traceback( self, etype, value, tb, tb_offset, number_of_lines_of_context) diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index fe62db0c353..e81bf221515 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -26,8 +26,6 @@ from unittest import TestCase -from IPython.testing.decorators import skipif - from IPython.extensions.autoreload import AutoreloadMagics from IPython.core.events import EventManager, pre_run_cell @@ -141,7 +139,6 @@ def pickle_get_current_class(obj): class TestAutoreload(Fixture): def test_reload_enums(self): - import enum mod_name, mod_fn = self.new_module(textwrap.dedent(""" from enum import Enum class MyEnum(Enum): diff --git a/IPython/lib/clipboard.py b/IPython/lib/clipboard.py index 1b8e756b5c6..316a8ab1f8a 100644 --- a/IPython/lib/clipboard.py +++ b/IPython/lib/clipboard.py @@ -32,15 +32,15 @@ def win32_clipboard_get(): win32clipboard.CloseClipboard() return text -def osx_clipboard_get(): +def osx_clipboard_get() -> str: """ Get the clipboard's text on OS X. """ p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'], stdout=subprocess.PIPE) - text, stderr = p.communicate() + bytes_, stderr = p.communicate() # Text comes in with old Mac \r line endings. Change them to \n. - text = text.replace(b'\r', b'\n') - text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING) + bytes_ = bytes_.replace(b'\r', b'\n') + text = py3compat.decode(bytes_) return text def tkinter_clipboard_get(): diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py index 37dd21abb90..bd8c01b2a75 100644 --- a/IPython/lib/deepreload.py +++ b/IPython/lib/deepreload.py @@ -7,13 +7,7 @@ imported from that module, which is useful when you're changing files deep inside a package. -To use this as your default reload function, type this for Python 2:: - - import __builtin__ - from IPython.lib import deepreload - __builtin__.reload = deepreload.reload - -Or this for Python 3:: +To use this as your default reload function, type this:: import builtins from IPython.lib import deepreload diff --git a/IPython/lib/display.py b/IPython/lib/display.py index e7c427cbe91..de31788ab97 100644 --- a/IPython/lib/display.py +++ b/IPython/lib/display.py @@ -130,7 +130,6 @@ def reload(self): @staticmethod def _make_wav(data, rate, normalize): """ Transform a numpy array to a PCM bytestring """ - import struct from io import BytesIO import wave @@ -145,7 +144,7 @@ def _make_wav(data, rate, normalize): waveobj.setframerate(rate) waveobj.setsampwidth(2) waveobj.setcomptype('NONE','NONE') - waveobj.writeframes(b''.join([struct.pack(' str: """Get the IPython directory for this platform and user. This uses the logic in `get_home_dir` to find the home directory @@ -28,10 +28,9 @@ def get_ipython_dir(): home_dir = get_home_dir() xdg_dir = get_xdg_dir() - # import pdb; pdb.set_trace() # dbg if 'IPYTHON_DIR' in env: - warn('The environment variable IPYTHON_DIR is deprecated. ' - 'Please use IPYTHONDIR instead.') + warn('The environment variable IPYTHON_DIR is deprecated since IPython 3.0. ' + 'Please use IPYTHONDIR instead.', DeprecationWarning) ipdir = env.get('IPYTHONDIR', env.get('IPYTHON_DIR', None)) if ipdir is None: # not set explicitly, use ~/.ipython @@ -67,11 +66,11 @@ def get_ipython_dir(): warn("IPython parent '{0}' is not a writable location," " using a temp directory.".format(parent)) ipdir = tempfile.mkdtemp() + assert isinstance(ipdir, str), "all path manipulation should be str(unicode), but are not." + return ipdir - return py3compat.cast_unicode(ipdir, fs_encoding) - -def get_ipython_cache_dir(): +def get_ipython_cache_dir() -> str: """Get the cache directory it is created if it does not exist.""" xdgdir = get_xdg_cache_dir() if xdgdir is None: @@ -82,13 +81,14 @@ def get_ipython_cache_dir(): elif not _writable_dir(xdgdir): return get_ipython_dir() - return py3compat.cast_unicode(ipdir, fs_encoding) + return ipdir -def get_ipython_package_dir(): +def get_ipython_package_dir() -> str: """Get the base directory where IPython itself is installed.""" ipdir = os.path.dirname(IPython.__file__) - return py3compat.cast_unicode(ipdir, fs_encoding) + assert isinstance(ipdir, str) + return ipdir def get_ipython_module_path(module_str): diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 0481624c42c..3d5b3768230 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -419,8 +419,8 @@ def process_image(self, decorator): # insert relative path to image file in source # as absolute path for Sphinx # sphinx expects a posix path, even on Windows - posix_path = pathlib.Path(savefig_dir,filename).as_posix() - outfile = '/' + os.path.relpath(posix_path, source_dir) + path = pathlib.Path(savefig_dir, filename) + outfile = '/' + path.relative_to(source_dir).as_posix() imagerows = ['.. image:: %s' % outfile] diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index ba5ce1bada8..ffd53c65e78 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -1,11 +1,13 @@ +import asyncio import signal import sys +import threading from IPython.core.debugger import Pdb from IPython.core.completer import IPCompleter from .ptutils import IPythonPTCompleter -from .shortcuts import suspend_to_bg, cursor_in_leading_ws +from .shortcuts import create_ipython_shortcuts, suspend_to_bg, cursor_in_leading_ws from prompt_toolkit.enums import DEFAULT_BUFFER from prompt_toolkit.filters import (Condition, has_focus, has_selection, @@ -39,23 +41,22 @@ def get_prompt_tokens(): global_namespace={}, parent=self.shell, ) - self._ptcomp = IPythonPTCompleter(compl) + # add a completer for all the do_ methods + methods_names = [m[3:] for m in dir(self) if m.startswith("do_")] - kb = KeyBindings() - supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP')) - kb.add('c-z', filter=supports_suspend)(suspend_to_bg) + def gen_comp(self, text): + return [m for m in methods_names if m.startswith(text)] + import types + newcomp = types.MethodType(gen_comp, compl) + compl.custom_matchers.insert(0, newcomp) + # end add completer. - if self.shell.display_completions == 'readlinelike': - kb.add('tab', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & vi_insert_mode | emacs_insert_mode - & ~cursor_in_leading_ws - ))(display_completions_like_readline) + self._ptcomp = IPythonPTCompleter(compl) options = dict( message=(lambda: PygmentsTokens(get_prompt_tokens())), editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()), - key_bindings=kb, + key_bindings=create_ipython_shortcuts(self.shell), history=self.shell.debugger_history, completer=self._ptcomp, enable_history_search=True, @@ -66,7 +67,8 @@ def get_prompt_tokens(): ) if not PTK3: - options['inputhook'] = self.inputhook + options['inputhook'] = self.shell.inputhook + self.pt_loop = asyncio.new_event_loop() self.pt_app = PromptSession(**options) def cmdloop(self, intro=None): @@ -79,6 +81,12 @@ def cmdloop(self, intro=None): if not self.use_rawinput: raise ValueError('Sorry ipdb does not support use_rawinput=False') + # In order to make sure that prompt, which uses asyncio doesn't + # interfere with applications in which it's used, we always run the + # prompt itself in a different thread (we can't start an event loop + # within an event loop). This new thread won't have any event loop + # running, and here we run our prompt-loop. + self.preloop() try: @@ -93,10 +101,27 @@ def cmdloop(self, intro=None): else: self._ptcomp.ipy_completer.namespace = self.curframe_locals self._ptcomp.ipy_completer.global_namespace = self.curframe.f_globals - try: - line = self.pt_app.prompt() # reset_current_buffer=True) - except EOFError: - line = 'EOF' + + # Run the prompt in a different thread. + line = '' + keyboard_interrupt = False + + def in_thread(): + nonlocal line, keyboard_interrupt + try: + line = self.pt_app.prompt() + except EOFError: + line = 'EOF' + except KeyboardInterrupt: + keyboard_interrupt = True + + th = threading.Thread(target=in_thread) + th.start() + th.join() + + if keyboard_interrupt: + raise KeyboardInterrupt + line = self.precmd(line) stop = self.onecmd(line) stop = self.postcmd(stop, line) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 26188aeba8a..b3bb5777450 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -102,7 +102,11 @@ class TerminalInteractiveShell(InteractiveShell): mime_renderers = Dict().tag(config=True) space_for_menu = Integer(6, help='Number of line at the bottom of the screen ' - 'to reserve for the completion menu' + 'to reserve for the tab completion menu, ' + 'search history, ...etc, the height of ' + 'these menus will at most this value. ' + 'Increase it is you prefer long and skinny ' + 'menus, decrease for short and wide.' ).tag(config=True) pt_app = None @@ -322,6 +326,7 @@ def prompt(): mouse_support=self.mouse_support, enable_open_in_editor=self.extra_open_editor_shortcuts, color_depth=self.color_depth, + tempfile_suffix=".py", **self._extra_prompt_options()) def _make_style_from_name_or_cls(self, name_or_cls): @@ -449,22 +454,28 @@ def prompt_for_code(self): else: default = '' - with patch_stdout(raw=True): - # In order to make sure that asyncio code written in the - # interactive shell doesn't interfere with the prompt, we run the - # prompt in a different event loop. - # If we don't do this, people could spawn coroutine with a - # while/true inside which will freeze the prompt. + # In order to make sure that asyncio code written in the + # interactive shell doesn't interfere with the prompt, we run the + # prompt in a different event loop. + # If we don't do this, people could spawn coroutine with a + # while/true inside which will freeze the prompt. + try: old_loop = asyncio.get_event_loop() - asyncio.set_event_loop(self.pt_loop) - try: + except RuntimeError: + # This happens when the user used `asyncio.run()`. + old_loop = None + + asyncio.set_event_loop(self.pt_loop) + try: + with patch_stdout(raw=True): text = self.pt_app.prompt( default=default, **self._extra_prompt_options()) - finally: - # Restore the original event loop. - asyncio.set_event_loop(old_loop) + finally: + # Restore the original event loop. + asyncio.set_event_loop(old_loop) + return text def enable_win_unicode_console(self): @@ -580,11 +591,22 @@ def enable_gui(self, gui=None): # For prompt_toolkit 3.0. We have to create an asyncio event loop with # this inputhook. if PTK3: - if self._inputhook: - from prompt_toolkit.eventloop import new_eventloop_with_inputhook + import asyncio + from prompt_toolkit.eventloop import new_eventloop_with_inputhook + + if gui == 'asyncio': + # When we integrate the asyncio event loop, run the UI in the + # same event loop as the rest of the code. don't use an actual + # input hook. (Asyncio is not made for nesting event loops.) + self.pt_loop = asyncio.get_event_loop() + + elif self._inputhook: + # If an inputhook was set, create a new asyncio event loop with + # this inputhook for the prompt. self.pt_loop = new_eventloop_with_inputhook(self._inputhook) else: - import asyncio + # When there's no inputhook, run the prompt in a separate + # asyncio event loop. self.pt_loop = asyncio.new_event_loop() # Run !system commands directly, not through pipes, so terminal programs diff --git a/IPython/terminal/prompts.py b/IPython/terminal/prompts.py index db1b751a026..3f5c07b980e 100644 --- a/IPython/terminal/prompts.py +++ b/IPython/terminal/prompts.py @@ -7,6 +7,7 @@ from prompt_toolkit.formatted_text import fragment_list_width, PygmentsTokens from prompt_toolkit.shortcuts import print_formatted_text +from prompt_toolkit.enums import EditingMode class Prompts(object): @@ -14,9 +15,14 @@ def __init__(self, shell): self.shell = shell def vi_mode(self): - if (getattr(self.shell.pt_app, 'editing_mode', None) == 'VI' + if (getattr(self.shell.pt_app, 'editing_mode', None) == EditingMode.VI and self.shell.prompt_includes_vi_mode): - return '['+str(self.shell.pt_app.app.vi_state.input_mode)[3:6]+'] ' + mode = str(self.shell.pt_app.app.vi_state.input_mode) + if mode.startswith('InputMode.'): + mode = mode[10:13].lower() + elif mode.startswith('vi-'): + mode = mode[3:6] + return '['+mode+'] ' return '' diff --git a/IPython/terminal/pt_inputhooks/asyncio.py b/IPython/terminal/pt_inputhooks/asyncio.py index e28603864f9..95cf194f866 100644 --- a/IPython/terminal/pt_inputhooks/asyncio.py +++ b/IPython/terminal/pt_inputhooks/asyncio.py @@ -28,6 +28,10 @@ """ import asyncio +from prompt_toolkit import __version__ as ptk_version + +PTK3 = ptk_version.startswith('3.') + # Keep reference to the original asyncio loop, because getting the event loop # within the input hook would return the other loop. @@ -35,6 +39,19 @@ def inputhook(context): + """ + Inputhook for asyncio event loop integration. + """ + # For prompt_toolkit 3.0, this input hook literally doesn't do anything. + # The event loop integration here is implemented in `interactiveshell.py` + # by running the prompt itself in the current asyncio loop. The main reason + # for this is that nesting asyncio event loops is unreliable. + if PTK3: + return + + # For prompt_toolkit 2.0, we can run the current asyncio event loop, + # because prompt_toolkit 2.0 uses a different event loop internally. + def stop(): loop.stop() @@ -44,3 +61,4 @@ def stop(): loop.run_forever() finally: loop.remove_reader(fileno) + diff --git a/IPython/terminal/pt_inputhooks/qt.py b/IPython/terminal/pt_inputhooks/qt.py index ce0a9da86bd..9f5e1b41dcd 100644 --- a/IPython/terminal/pt_inputhooks/qt.py +++ b/IPython/terminal/pt_inputhooks/qt.py @@ -25,6 +25,7 @@ def inputhook(context): 'variable. Deactivate Qt5 code.' ) return + QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling) _appref = app = QtGui.QApplication([" "]) event_loop = QtCore.QEventLoop(app) @@ -44,7 +45,7 @@ def inputhook(context): QtCore.QSocketNotifier.Read) try: # connect the callback we care about before we turn it on - notifier.activated.connect(event_loop.exit) + notifier.activated.connect(lambda: event_loop.exit()) notifier.setEnabled(True) # only start the event loop we are not already flipped if not context.input_is_ready(): diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index 618f092b4f9..a0f4442c771 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -177,11 +177,13 @@ def inputhook_wxphoenix(context): # Use a wx.Timer to periodically check whether input is ready - as soon as # it is, we exit the main loop + timer = wx.Timer() + def poll(ev): if context.input_is_ready(): + timer.Stop() app.ExitMainLoop() - timer = wx.Timer() timer.Start(poll_interval) timer.Bind(wx.EVT_TIMER, poll) diff --git a/IPython/terminal/ptutils.py b/IPython/terminal/ptutils.py index 4f21cb04e35..ed7ad45eb45 100644 --- a/IPython/terminal/ptutils.py +++ b/IPython/terminal/ptutils.py @@ -23,7 +23,7 @@ _completion_sentinel = object() -def _elide(string, *, min_elide=30): +def _elide_point(string:str, *, min_elide=30)->str: """ If a string is long enough, and has at least 3 dots, replace the middle part with ellipses. @@ -42,6 +42,8 @@ def _elide(string, *, min_elide=30): object_parts = string.split('.') file_parts = string.split(os.sep) + if file_parts[-1] == '': + file_parts.pop() if len(object_parts) > 3: return '{}.{}\N{HORIZONTAL ELLIPSIS}{}.{}'.format(object_parts[0], object_parts[1][0], object_parts[-2][-1], object_parts[-1]) @@ -51,6 +53,26 @@ def _elide(string, *, min_elide=30): return string +def _elide_typed(string:str, typed:str, *, min_elide:int=30)->str: + """ + Elide the middle of a long string if the beginning has already been typed. + """ + + if len(string) < min_elide: + return string + cut_how_much = len(typed)-3 + if cut_how_much < 7: + return string + if string.startswith(typed) and len(string)> len(typed): + return f"{string[:3]}\N{HORIZONTAL ELLIPSIS}{string[cut_how_much:]}" + return string + +def _elide(string:str, typed:str, min_elide=30)->str: + return _elide_typed( + _elide_point(string, min_elide=min_elide), + typed, min_elide=min_elide) + + def _adjust_completion_text_based_on_context(text, body, offset): if text.endswith('=') and len(body) > offset and body[offset] == '=': @@ -87,7 +109,11 @@ def get_completions(self, document, complete_event): cursor_col = document.cursor_position_col cursor_position = document.cursor_position offset = cursor_to_position(body, cursor_row, cursor_col) - yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + try: + yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + except Exception as e: + from traceback import print_tb + print_tb(e) @staticmethod def _get_completions(body, offset, cursor_position, ipyc): @@ -126,9 +152,9 @@ def _get_completions(body, offset, cursor_position, ipyc): adjusted_text = _adjust_completion_text_based_on_context(c.text, body, offset) if c.type == 'function': - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()'), display_meta=c.type+c.signature) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()', body[c.start:c.end]), display_meta=c.type+c.signature) else: - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text), display_meta=c.type) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text, body[c.start:c.end]), display_meta=c.type) class IPythonPTLexer(Lexer): """ diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index e44e34277e2..a23fa091a0e 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -130,8 +130,10 @@ def newline_or_execute(event): # if all we have after the cursor is whitespace: reformat current text # before cursor after_cursor = d.text[d.cursor_position:] + reformatted = False if not after_cursor.strip(): reformat_text_before_cursor(b, d, shell) + reformatted = True if not (d.on_last_line or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() ): @@ -142,7 +144,8 @@ def newline_or_execute(event): return if (status != 'incomplete') and b.accept_handler: - reformat_text_before_cursor(b, d, shell) + if not reformatted: + reformat_text_before_cursor(b, d, shell) b.validate_and_handle() else: if shell.autoindent: @@ -250,7 +253,6 @@ def newline_autoindent(event): def open_input_in_editor(event): - event.app.current_buffer.tempfile_suffix = ".py" event.app.current_buffer.open_in_editor() diff --git a/IPython/terminal/tests/test_interactivshell.py b/IPython/terminal/tests/test_interactivshell.py index 9651a224dad..640e5d482ae 100644 --- a/IPython/terminal/tests/test_interactivshell.py +++ b/IPython/terminal/tests/test_interactivshell.py @@ -5,6 +5,7 @@ import sys import unittest +import os from IPython.core.inputtransformer import InputTransformer from IPython.testing import tools as tt @@ -16,10 +17,32 @@ class TestElide(unittest.TestCase): def test_elide(self): - _elide('concatenate((a1, a2, ...), axis') # do not raise - _elide('concatenate((a1, a2, ..), . axis') # do not raise - nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh'), 'aaaa.b…g.hhhhhh') - + _elide('concatenate((a1, a2, ...), axis', '') # do not raise + _elide('concatenate((a1, a2, ..), . axis', '') # do not raise + nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh',''), 'aaaa.b…g.hhhhhh') + + test_string = os.sep.join(['', 10*'a', 10*'b', 10*'c', '']) + expect_stirng = os.sep + 'a' + '\N{HORIZONTAL ELLIPSIS}' + 'b' + os.sep + 10*'c' + nt.assert_equal(_elide(test_string, ''), expect_stirng) + + def test_elide_typed_normal(self): + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick brown fox', min_elide=10), 'the…fox jumped over the lazy dog') + + + def test_elide_typed_short_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the', min_elide=10), 'the quick brown fox jumped over the lazy dog') + + def test_elide_typed_no_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + # here we typed red instead of brown + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick red fox', min_elide=10), 'the quick brown fox jumped over the lazy dog') class TestContextAwareCompletion(unittest.TestCase): diff --git a/IPython/testing/ipunittest.py b/IPython/testing/ipunittest.py index 56146d1de26..5a940a5fe91 100644 --- a/IPython/testing/ipunittest.py +++ b/IPython/testing/ipunittest.py @@ -155,7 +155,7 @@ def test(self): # failed should contain at most one item. More than that # is a case we can't handle and should error out on if len(failed) > 1: - err = "Invalid number of test results:" % failed + err = "Invalid number of test results: %s" % failed raise ValueError(err) # Report a normal failure. self.fail('failed doctests: %s' % str(failed[0])) diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py new file mode 100644 index 00000000000..026415a57a4 --- /dev/null +++ b/IPython/tests/cve.py @@ -0,0 +1,56 @@ +""" +Test that CVEs stay fixed. +""" + +from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory +from pathlib import Path +import random +import sys +import os +import string +import subprocess +import time + +def test_cve_2022_21699(): + """ + Here we test CVE-2022-21699. + + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, + checking for the value. + + + + """ + + dangerous_profile_dir = Path('profile_default') + + dangerous_startup_dir = dangerous_profile_dir / 'startup' + dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + + with TemporaryWorkingDirectory() as t: + dangerous_startup_dir.mkdir(parents=True) + (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + # 1 sec to make sure FS is flushed. + #time.sleep(1) + cmd = [sys.executable,'-m', 'IPython'] + env = os.environ.copy() + env['IPY_TEST_SIMPLE_PROMPT'] = '1' + + + # First we fake old behavior, making sure the profile is/was actually dangerous + p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") + assert dangerous_expected in out_dangerous.decode() + + # Now that we know it _would_ have been dangerous, we test it's not loaded + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate(b"exit\r") + assert b'IPython' in out + assert dangerous_expected not in out.decode() + assert err == b'' + + + diff --git a/IPython/utils/_process_posix.py b/IPython/utils/_process_posix.py index f3f93f774d6..a11cad7697c 100644 --- a/IPython/utils/_process_posix.py +++ b/IPython/utils/_process_posix.py @@ -59,11 +59,12 @@ class ProcessHandler(object): @property def sh(self): - if self._sh is None: - self._sh = pexpect.which('sh') + if self._sh is None: + shell_name = os.environ.get("SHELL", "sh") + self._sh = pexpect.which(shell_name) if self._sh is None: - raise OSError('"sh" shell not found') - + raise OSError('"{}" shell not found'.format(shell_name)) + return self._sh def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None): diff --git a/IPython/utils/_process_win32.py b/IPython/utils/_process_win32.py index fb17e4c5d37..6d05bdaa12e 100644 --- a/IPython/utils/_process_win32.py +++ b/IPython/utils/_process_win32.py @@ -18,10 +18,12 @@ import os import sys import ctypes +import time from ctypes import c_int, POINTER from ctypes.wintypes import LPCWSTR, HLOCAL -from subprocess import STDOUT +from subprocess import STDOUT, TimeoutExpired +from threading import Thread # our own imports from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split @@ -93,15 +95,29 @@ def _find_cmd(cmd): def _system_body(p): """Callback for _system.""" enc = DEFAULT_ENCODING - for line in read_no_interrupt(p.stdout).splitlines(): - line = line.decode(enc, 'replace') - print(line, file=sys.stdout) - for line in read_no_interrupt(p.stderr).splitlines(): - line = line.decode(enc, 'replace') - print(line, file=sys.stderr) - # Wait to finish for returncode - return p.wait() + def stdout_read(): + for line in read_no_interrupt(p.stdout).splitlines(): + line = line.decode(enc, 'replace') + print(line, file=sys.stdout) + + def stderr_read(): + for line in read_no_interrupt(p.stderr).splitlines(): + line = line.decode(enc, 'replace') + print(line, file=sys.stderr) + + Thread(target=stdout_read).start() + Thread(target=stderr_read).start() + + # Wait to finish for returncode. Unfortunately, Python has a bug where + # wait() isn't interruptible (https://bugs.python.org/issue28168) so poll in + # a loop instead of just doing `return p.wait()`. + while True: + result = p.poll() + if result is None: + time.sleep(0.01) + else: + return result def system(cmd): @@ -116,9 +132,7 @@ def system(cmd): Returns ------- - None : we explicitly do NOT return the subprocess status code, as this - utility is meant to be used extensively in IPython, where any return value - would trigger :func:`sys.displayhook` calls. + int : child process' exit code. """ # The controller provides interactivity with both # stdin and stdout diff --git a/IPython/utils/daemonize.py b/IPython/utils/daemonize.py index 0de3852c78c..44b4a2832e0 100644 --- a/IPython/utils/daemonize.py +++ b/IPython/utils/daemonize.py @@ -1,4 +1,4 @@ from warnings import warn -warn("IPython.utils.daemonize has moved to ipyparallel.apps.daemonize", stacklevel=2) +warn("IPython.utils.daemonize has moved to ipyparallel.apps.daemonize since IPython 4.0", DeprecationWarning, stacklevel=2) from ipyparallel.apps.daemonize import daemonize diff --git a/IPython/utils/encoding.py b/IPython/utils/encoding.py index 387a24700cf..69a319ef0ef 100644 --- a/IPython/utils/encoding.py +++ b/IPython/utils/encoding.py @@ -44,7 +44,7 @@ def getdefaultencoding(prefer_stream=True): Then fall back on locale.getpreferredencoding(), which should be a sensible platform default (that respects LANG environment), and finally to sys.getdefaultencoding() which is the most conservative option, - and usually ASCII on Python 2 or UTF8 on Python 3. + and usually UTF8 as of Python 3. """ enc = None if prefer_stream: diff --git a/IPython/utils/frame.py b/IPython/utils/frame.py index 11dab31f8d0..74c6d4197f4 100644 --- a/IPython/utils/frame.py +++ b/IPython/utils/frame.py @@ -15,7 +15,6 @@ #----------------------------------------------------------------------------- import sys -from IPython.utils import py3compat #----------------------------------------------------------------------------- # Code diff --git a/IPython/utils/openpy.py b/IPython/utils/openpy.py index 3046c31587d..c90d2b53a30 100644 --- a/IPython/utils/openpy.py +++ b/IPython/utils/openpy.py @@ -66,8 +66,7 @@ def read_py_file(filename, skip_encoding_cookie=True): The path to the file to read. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first - two lines, that line will be excluded from the output - compiling a - unicode string with an encoding declaration is a SyntaxError in Python 2. + two lines, that line will be excluded from the output. Returns ------- @@ -91,8 +90,7 @@ def read_py_url(url, errors='replace', skip_encoding_cookie=True): bytes.decode(), but here 'replace' is the default. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first - two lines, that line will be excluded from the output - compiling a - unicode string with an encoding declaration is a SyntaxError in Python 2. + two lines, that line will be excluded from the output. Returns ------- diff --git a/IPython/utils/path.py b/IPython/utils/path.py index f677f1c7cd0..0fb6144e19f 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -15,13 +15,11 @@ from warnings import warn from IPython.utils.process import system -from IPython.utils import py3compat from IPython.utils.decorators import undoc #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- - fs_encoding = sys.getfilesystemencoding() def _writable_dir(path): @@ -169,7 +167,7 @@ class HomeDirError(Exception): pass -def get_home_dir(require_writable=False): +def get_home_dir(require_writable=False) -> str: """Return the 'home' directory, as a unicode string. Uses os.path.expanduser('~'), and checks for writability. @@ -197,21 +195,18 @@ def get_home_dir(require_writable=False): if not _writable_dir(homedir) and os.name == 'nt': # expanduser failed, use the registry to get the 'My Documents' folder. try: - try: - import winreg as wreg # Py 3 - except ImportError: - import _winreg as wreg # Py 2 - key = wreg.OpenKey( + import winreg as wreg + with wreg.OpenKey( wreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" - ) - homedir = wreg.QueryValueEx(key,'Personal')[0] - key.Close() + ) as key: + homedir = wreg.QueryValueEx(key,'Personal')[0] except: pass if (not require_writable) or _writable_dir(homedir): - return py3compat.cast_unicode(homedir, fs_encoding) + assert isinstance(homedir, str), "Homedir shoudl be unicode not bytes" + return homedir else: raise HomeDirError('%s is not a writable dir, ' 'set $HOME environment variable to override' % homedir) @@ -229,7 +224,8 @@ def get_xdg_dir(): # use ~/.config if empty OR not set xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config') if xdg and _writable_dir(xdg): - return py3compat.cast_unicode(xdg, fs_encoding) + assert isinstance(xdg, str) + return xdg return None @@ -247,38 +243,39 @@ def get_xdg_cache_dir(): # use ~/.cache if empty OR not set xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache') if xdg and _writable_dir(xdg): - return py3compat.cast_unicode(xdg, fs_encoding) + assert isinstance(xdg, str) + return xdg return None @undoc def get_ipython_dir(): - warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_dir return get_ipython_dir() @undoc def get_ipython_cache_dir(): - warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_cache_dir return get_ipython_cache_dir() @undoc def get_ipython_package_dir(): - warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_package_dir return get_ipython_package_dir() @undoc def get_ipython_module_path(module_str): - warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_module_path return get_ipython_module_path(module_str) @undoc def locate_profile(profile='default'): - warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import locate_profile return locate_profile(profile=profile) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index e5716650392..c7587873005 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -13,8 +13,6 @@ from .encoding import DEFAULT_ENCODING -def no_code(x, encoding=None): - return x def decode(s, encoding=None): encoding = encoding or DEFAULT_ENCODING @@ -44,7 +42,7 @@ def buffer_to_bytes(buf): def _modify_str_or_docstring(str_change_func): @functools.wraps(str_change_func) def wrapper(func_or_str): - if isinstance(func_or_str, string_types): + if isinstance(func_or_str, (str,)): func = None doc = func_or_str else: @@ -66,17 +64,12 @@ def safe_unicode(e): safe to call unicode() on. """ try: - return unicode_type(e) - except UnicodeError: - pass - - try: - return str_to_unicode(str(e)) + return str(e) except UnicodeError: pass try: - return str_to_unicode(repr(e)) + return repr(e) except UnicodeError: pass @@ -156,16 +149,6 @@ def input(prompt=''): builtin_mod_name = "builtins" import builtins as builtin_mod -str_to_unicode = no_code -unicode_to_str = no_code -str_to_bytes = encode -bytes_to_str = decode -cast_bytes_py2 = no_code -cast_unicode_py2 = no_code -buffer_to_bytes_py2 = no_code - -string_types = (str,) -unicode_type = str which = shutil.which @@ -174,9 +157,6 @@ def isidentifier(s, dotted=False): return all(isidentifier(a) for a in s.split(".")) return s.isidentifier() -xrange = range -def iteritems(d): return iter(d.items()) -def itervalues(d): return iter(d.values()) getcwd = os.getcwd MethodType = types.MethodType @@ -189,9 +169,6 @@ def execfile(fname, glob, loc=None, compiler=None): # Refactor print statements in doctests. _print_statement_re = re.compile(r"\bprint (?P.*)$", re.MULTILINE) -def _print_statement_sub(match): - expr = match.groups('expr') - return "print(%s)" % expr # Abstract u'abc' syntax: @_modify_str_or_docstring @@ -201,50 +178,14 @@ def u_format(s): Accepts a string or a function, so it can be used as a decorator.""" return s.format(u='') -def get_closure(f): - """Get a function's closure attribute""" - return f.__closure__ - PY2 = not PY3 PYPY = platform.python_implementation() == "PyPy" +# Cython still rely on that as a Dec 28 2019 +# See https://github.com/cython/cython/pull/3291 and +# https://github.com/ipython/ipython/issues/12068 +def no_code(x, encoding=None): + return x +unicode_to_str = cast_bytes_py2 = no_code -def annotate(**kwargs): - """Python 3 compatible function annotation for Python 2.""" - if not kwargs: - raise ValueError('annotations must be provided as keyword arguments') - def dec(f): - if hasattr(f, '__annotations__'): - for k, v in kwargs.items(): - f.__annotations__[k] = v - else: - f.__annotations__ = kwargs - return f - return dec - - -# Parts below taken from six: -# Copyright (c) 2010-2013 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - return meta("_NewBase", bases, {}) diff --git a/IPython/utils/sysinfo.py b/IPython/utils/sysinfo.py index 7c29ed028d5..07d14fd8a48 100644 --- a/IPython/utils/sysinfo.py +++ b/IPython/utils/sysinfo.py @@ -55,10 +55,10 @@ def pkg_commit_hash(pkg_path): return "installation", _sysinfo.commit # maybe we are in a repository - proc = subprocess.Popen('git rev-parse --short HEAD', + proc = subprocess.Popen('git rev-parse --short HEAD'.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=pkg_path, shell=True) + cwd=pkg_path) repo_commit, _ = proc.communicate() if repo_commit: return 'repository', repo_commit.strip().decode('ascii') diff --git a/IPython/utils/tests/test_io.py b/IPython/utils/tests/test_io.py index e6af8c25a43..83367fa9e9f 100644 --- a/IPython/utils/tests/test_io.py +++ b/IPython/utils/tests/test_io.py @@ -5,9 +5,6 @@ # Distributed under the terms of the Modified BSD License. -import io as stdlib_io -import os.path -import stat import sys from io import StringIO @@ -16,9 +13,7 @@ import nose.tools as nt -from IPython.testing.decorators import skipif, skip_win32 from IPython.utils.io import IOStream, Tee, capture_output -from IPython.utils.tempdir import TemporaryDirectory def test_tee_simple(): diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index b34f9ca926c..74f21c3b50a 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -21,9 +21,9 @@ from IPython import paths from IPython.testing import decorators as dec from IPython.testing.decorators import (skip_if_not_win32, skip_win32, - onlyif_unicode_paths, skipif, + onlyif_unicode_paths, skip_win32_py38,) -from IPython.testing.tools import make_tempfile, AssertPrints +from IPython.testing.tools import make_tempfile from IPython.utils import path from IPython.utils.tempdir import TemporaryDirectory @@ -171,8 +171,12 @@ def test_get_home_dir_8(): env.pop(key, None) class key: + def __enter__(self): + pass def Close(self): pass + def __exit__(*args, **kwargs): + pass with patch.object(wreg, 'OpenKey', return_value=key()), \ patch.object(wreg, 'QueryValueEx', return_value=[abspath(HOME_TEST_DIR)]): diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index 956b2abbae2..fbc000c3eeb 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -15,14 +15,19 @@ #----------------------------------------------------------------------------- import sys +import signal import os -from unittest import TestCase +import time +from _thread import interrupt_main # Py 3 +import threading +from unittest import SkipTest import nose.tools as nt from IPython.utils.process import (find_cmd, FindCmdError, arg_split, system, getoutput, getoutputerror, get_output_error_code) +from IPython.utils.capture import capture_output from IPython.testing import decorators as dec from IPython.testing import tools as tt @@ -108,6 +113,49 @@ def test_system_quotes(self): status = system('%s -c "import sys"' % python) self.assertEqual(status, 0) + def assert_interrupts(self, command): + """ + Interrupt a subprocess after a second. + """ + if threading.main_thread() != threading.current_thread(): + raise nt.SkipTest("Can't run this test if not in main thread.") + + # Some tests can overwrite SIGINT handler (by using pdb for example), + # which then breaks this test, so just make sure it's operating + # normally. + signal.signal(signal.SIGINT, signal.default_int_handler) + + def interrupt(): + # Wait for subprocess to start: + time.sleep(0.5) + interrupt_main() + + threading.Thread(target=interrupt).start() + start = time.time() + try: + result = command() + except KeyboardInterrupt: + # Success! + pass + end = time.time() + self.assertTrue( + end - start < 2, "Process didn't die quickly: %s" % (end - start) + ) + return result + + def test_system_interrupt(self): + """ + When interrupted in the way ipykernel interrupts IPython, the + subprocess is interrupted. + """ + def command(): + return system('%s -c "import time; time.sleep(5)"' % python) + + status = self.assert_interrupts(command) + self.assertNotEqual( + status, 0, "The process wasn't interrupted. Status: %s" % (status,) + ) + def test_getoutput(self): out = getoutput('%s "%s"' % (python, self.fname)) # we can't rely on the order the line buffered streams are flushed @@ -132,7 +180,7 @@ def test_getoutput_error(self): out, err = getoutputerror('%s "%s"' % (python, self.fname)) self.assertEqual(out, 'on stdout') self.assertEqual(err, 'on stderr') - + def test_get_output_error_code(self): quiet_exit = '%s -c "import sys; sys.exit(1)"' % python out, err, code = get_output_error_code(quiet_exit) @@ -143,3 +191,5 @@ def test_get_output_error_code(self): self.assertEqual(out, 'on stdout') self.assertEqual(err, 'on stderr') self.assertEqual(code, 0) + + diff --git a/IPython/utils/text.py b/IPython/utils/text.py index e844203ca0e..14405ede1f7 100644 --- a/IPython/utils/text.py +++ b/IPython/utils/text.py @@ -538,7 +538,7 @@ class FullEvalFormatter(Formatter): """ # copied from Formatter._vformat with minor changes to allow eval # and replace the format_spec code with slicing - def vformat(self, format_string, args, kwargs): + def vformat(self, format_string:str, args, kwargs)->str: result = [] for literal_text, field_name, format_spec, conversion in \ self.parse(format_string): @@ -566,7 +566,7 @@ def vformat(self, format_string, args, kwargs): # format the object and append to the result result.append(self.format_field(obj, '')) - return ''.join(py3compat.cast_unicode(s) for s in result) + return ''.join(result) class DollarFormatter(FullEvalFormatter): diff --git a/MANIFEST.in b/MANIFEST.in index dee45a2c408..d47a16d42f8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,6 +24,8 @@ graft docs exclude docs/\#* exclude docs/man/*.1.gz +exclude .git-blame-ignore-revs + # Examples graft examples diff --git a/README.rst b/README.rst index 3e13610a443..9c0c89ec8e0 100644 --- a/README.rst +++ b/README.rst @@ -129,3 +129,18 @@ project that you might want to use: - `mypython `_ - `ptpython and ptipython ` - `xonsh ` + +Ignoring commits with git blame.ignoreRevsFile +============================================== + +As of git 2.23, it is possible to make formatting changes without breaking +``git blame``. See the `git documentation +`_ +for more details. + +To use this feature you must: + +- Install git >= 2.23 +- Configure your local git repo by running: + - POSIX: ``tools\configure-git-blame-ignore-revs.sh`` + - Windows: ``tools\configure-git-blame-ignore-revs.bat`` diff --git a/appveyor.yml b/appveyor.yml index 5ed18735ff0..d20effdf815 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,17 +4,6 @@ matrix: environment: matrix: - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python37" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "32" - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7.x" diff --git a/docs/source/conf.py b/docs/source/conf.py index 8a7a6098c3f..5012da2694e 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -121,6 +121,29 @@ def is_stable(extra): numpydoc_class_members_toctree = False warning_is_error = True +import logging + +class ConfigtraitFilter(logging.Filter): + """ + This is a filter to remove in sphinx 3+ the error about config traits being duplicated. + + As we autogenerate configuration traits from, subclasses have lots of + duplication and we want to silence them. Indeed we build on travis with + warnings-as-error set to True, so those duplicate items make the build fail. + """ + + def filter(self, record): + if record.args and record.args[0] == 'configtrait' and 'duplicate' in record.msg: + return False + return True + +ct_filter = ConfigtraitFilter() + +import sphinx.util +logger = sphinx.util.logging.getLogger('sphinx.domains.std').logger + +logger.addFilter(ct_filter) + # The default replacements for |version| and |release|, also used in various # other places throughout the built documents. # @@ -223,9 +246,8 @@ def is_stable(extra): htmlhelp_basename = 'ipythondoc' intersphinx_mapping = {'python': ('https://docs.python.org/3/', None), - 'rpy2': ('https://rpy2.readthedocs.io/en/version_2.8.x/', None), + 'rpy2': ('https://rpy2.github.io/doc/latest/html/', None), 'jupyterclient': ('https://jupyter-client.readthedocs.io/en/latest/', None), - 'ipyparallel': ('https://ipyparallel.readthedocs.io/en/latest/', None), 'jupyter': ('https://jupyter.readthedocs.io/en/latest/', None), 'jedi': ('https://jedi.readthedocs.io/en/latest/', None), 'traitlets': ('https://traitlets.readthedocs.io/en/latest/', None), diff --git a/docs/source/config/details.rst b/docs/source/config/details.rst index 6685b14d8f6..9e63232d81d 100644 --- a/docs/source/config/details.rst +++ b/docs/source/config/details.rst @@ -247,7 +247,7 @@ VI input mode to ``Normal`` when in insert mode:: For more information on filters and what you can do with the ``event`` object, `see the prompt_toolkit docs -`__. +`__. Enter to execute diff --git a/docs/source/config/intro.rst b/docs/source/config/intro.rst index 115315c9ddb..6dccde1348c 100644 --- a/docs/source/config/intro.rst +++ b/docs/source/config/intro.rst @@ -11,48 +11,49 @@ Many of IPython's classes have configurable attributes (see :doc:`options/index` for the list). These can be configured in several ways. -Python config files -------------------- +Python configuration files +-------------------------- -To create the blank config files, run:: +To create the blank configuration files, run:: ipython profile create [profilename] If you leave out the profile name, the files will be created for the -``default`` profile (see :ref:`profiles`). These will typically be -located in :file:`~/.ipython/profile_default/`, and will be named -:file:`ipython_config.py`, :file:`ipython_notebook_config.py`, etc. -The settings in :file:`ipython_config.py` apply to all IPython commands. +``default`` profile (see :ref:`profiles`). These will typically be located in +:file:`~/.ipython/profile_default/`, and will be named +:file:`ipython_config.py`, for historical reasons you may also find files +named with IPython prefix instead of Jupyter: +:file:`ipython_notebook_config.py`, etc. The settings in +:file:`ipython_config.py` apply to all IPython commands. -The files typically start by getting the root config object:: - - c = get_config() +By default, configuration files are fully featured Python scripts that can +execute arbitrary code, the main usage is to set value on the configuration +object ``c`` which exist in your configuration file. You can then configure class attributes like this:: c.InteractiveShell.automagic = False Be careful with spelling--incorrect names will simply be ignored, with -no error. +no error. -To add to a collection which may have already been defined elsewhere, -you can use methods like those found on lists, dicts and sets: append, -extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like -extend, but at the front), add and update (which works both for dicts -and sets):: +To add to a collection which may have already been defined elsewhere or have +default values, you can use methods like those found on lists, dicts and +sets: append, extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like +extend, but at the front), add and update (which works both for dicts and +sets):: c.InteractiveShellApp.extensions.append('Cython') .. versionadded:: 2.0 list, dict and set methods for config values -Example config file -``````````````````` +Example configuration file +`````````````````````````` :: # sample ipython_config.py - c = get_config() c.TerminalIPythonApp.display_banner = True c.InteractiveShellApp.log_level = 20 @@ -78,6 +79,38 @@ Example config file ('la', 'ls -al') ] +JSON Configuration files +------------------------ + +In case where executability of configuration can be problematic, or +configurations need to be modified programmatically, IPython also support a +limited set of functionalities via ``.json`` configuration files. + +You can defined most of the configuration options via a json object which +hierarchy represent the value you would normally set on the ``c`` object of +``.py`` configuration files. The following ``ipython_config.json`` file:: + + { + "InteractiveShell": { + "colors": "LightBG", + "editor": "nano" + }, + "InteractiveShellApp": { + "extensions": [ + "myextension" + ] + } + } + +Is equivalent to the following ``ipython_config.py``:: + + c.InteractiveShellApp.extensions = [ + 'myextension' + ] + + c.InteractiveShell.colors = 'LightBG' + c.InteractiveShell.editor = 'nano' + Command line arguments ---------------------- @@ -94,7 +127,7 @@ Many frequently used options have short aliases and flags, such as To see all of these abbreviated options, run:: ipython --help - ipython notebook --help + jupyter notebook --help # etc. Options specified at the command line, in either format, override @@ -163,3 +196,38 @@ the directory :file:`~/.ipython/` by default. To see where IPython is looking for the IPython directory, use the command ``ipython locate``, or the Python function :func:`IPython.paths.get_ipython_dir`. + + +Systemwide configuration +======================== + +It can be useful to deploy systemwide ipython or ipykernel configuration +when managing environment for many users. At startup time IPython and +IPykernel will search for configuration file in multiple systemwide +locations, mainly: + + - ``/etc/ipython/`` + - ``/usr/local/etc/ipython/`` + +When the global install is a standalone python distribution it may also +search in distribution specific location, for example: + + - ``$ANACONDA_LOCATION/etc/ipython/`` + +In those locations, Terminal IPython will look for a file called +``ipython_config.py`` and ``ipython_config.json``, ipykernel will look for +``ipython_kernel_config.py`` and ``ipython_kernel.json``. + +Configuration files are loaded in order and merged with configuration on +later location taking precedence on earlier locations (that is to say a user +can overwrite a systemwide configuration option). + +You can see all locations in which IPython is looking for configuration files +by starting ipython in debug mode:: + + $ ipython --debug -c 'exit()' + +Identically with ipykernel though the command is currently blocking until +this process is killed with ``Ctrl-\``:: + + $ python -m ipykernel --debug diff --git a/docs/source/coredev/index.rst b/docs/source/coredev/index.rst index 60f1cb0fdfd..ee1eadb9b1e 100644 --- a/docs/source/coredev/index.rst +++ b/docs/source/coredev/index.rst @@ -80,6 +80,13 @@ for the release you are actually making:: VERSION=5.0.0 BRANCH=master +For `reproducibility of builds `_, +we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value +of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You +should be able to use ``date +%s`` to get a formatted timestamp:: + + SOURCE_DATE_EPOCH=$(date +%s) + 2. Create GitHub stats and finish release note ---------------------------------------------- @@ -229,6 +236,16 @@ uploading them to PyPI. We do not use an universal wheel as each wheel installs an ``ipython2`` or ``ipython3`` script, depending on the version of Python it is built for. Using an universal wheel would prevent this. +Check the shasum of files with:: + + shasum -a 256 dist/* + +and takes notes of them you might need them to update the conda-forge recipes. +Rerun the command and check the hash have not changed:: + + ./tools/release + shasum -a 256 dist/* + Use the following to actually upload the result of the build:: ./tools/release upload diff --git a/docs/source/development/wrapperkernels.rst b/docs/source/development/wrapperkernels.rst index eb0a0488807..d734c30ee99 100644 --- a/docs/source/development/wrapperkernels.rst +++ b/docs/source/development/wrapperkernels.rst @@ -116,7 +116,7 @@ You can override a number of other methods to improve the functionality of your kernel. All of these methods should return a dictionary as described in the relevant section of the :doc:`messaging spec `. -.. class:: MyKernel +.. class:: MyBetterKernel .. method:: do_complete(code, cusor_pos) diff --git a/docs/source/interactive/autoawait.rst b/docs/source/interactive/autoawait.rst index f87379d2ca0..e4ed965a13e 100644 --- a/docs/source/interactive/autoawait.rst +++ b/docs/source/interactive/autoawait.rst @@ -8,7 +8,7 @@ Asynchronous in REPL: Autoawait This feature is experimental and behavior can change between python and IPython version without prior deprecation. -Starting with IPython 7.0, and when user Python 3.6 and above, IPython offer the +Starting with IPython 7.0, and when using Python 3.6 and above, IPython offer the ability to run asynchronous code from the REPL. Constructs which are :exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython. @@ -19,7 +19,7 @@ will differ between IPython, IPykernel and their versions. When a supported library is used, IPython will automatically allow Futures and Coroutines in the REPL to be ``await`` ed. This will happen if an :ref:`await -` (or any other async constructs like async-with, async-for) is use at +` (or any other async constructs like async-with, async-for) is used at top level scope, or if any structure valid only in `async def `_ function context are present. For example, the following being a syntax error in the @@ -73,7 +73,7 @@ By default IPython will assume integration with Python's provided :mod:`asyncio`, but integration with other libraries is provided. In particular we provide experimental integration with the ``curio`` and ``trio`` library. -You can switch current integration by using the +You can switch the current integration by using the ``c.InteractiveShell.loop_runner`` option or the ``autoawait `` magic. @@ -118,7 +118,7 @@ to your code. When using command line IPython, the default loop (or runner) does not process in the background, so top level asynchronous code must finish for the REPL to -allow you to enter more code. As with usual Python semantic, the awaitables are +allow you to enter more code. As with usual Python semantics, the awaitables are started only when awaited for the first time. That is to say, in first example, no network request is done between ``In[1]`` and ``In[2]``. @@ -131,8 +131,8 @@ a loop to run. By default IPython will use a fake coroutine runner which should allow ``IPython.embed()`` to be nested. Though this will prevent usage of the :magic:`%autoawait` feature when using IPython embed. -You can set explicitly a coroutine runner for ``embed()`` if you desire to run -asynchronous code, the exact behavior is though undefined. +You can set a coroutine runner explicitly for ``embed()`` if you want to run +asynchronous code, though the exact behavior is undefined. Effects on Magics ----------------- @@ -140,14 +140,14 @@ Effects on Magics A couple of magics (``%%timeit``, ``%timeit``, ``%%time``, ``%%prun``) have not yet been updated to work with asynchronous code and will raise syntax errors when trying to use top-level ``await``. We welcome any contribution to help fix -those, and extra cases we haven't caught yet. We hope for better support in Cor +those, and extra cases we haven't caught yet. We hope for better support in Core Python for top-level Async code. Internals --------- As running asynchronous code is not supported in interactive REPL (as of Python -3.7) we have to rely to a number of complex workaround and heuristic to allow +3.7) we have to rely to a number of complex workarounds and heuristics to allow this to happen. It is interesting to understand how this works in order to comprehend potential bugs, or provide a custom runner. @@ -179,16 +179,16 @@ significant overhead to this kind of code. By default the generated coroutine function will be consumed by Asyncio's ``loop_runner = asyncio.get_evenloop().run_until_complete()`` method if ``async`` mode is deemed necessary, otherwise the coroutine will just be -exhausted in a simple runner. It is though possible to change the default +exhausted in a simple runner. It is possible, though, to change the default runner. A loop runner is a *synchronous* function responsible from running a coroutine object. -The runner is responsible from ensuring that ``coroutine`` run to completion, -and should return the result of executing the coroutine. Let's write a +The runner is responsible for ensuring that ``coroutine`` runs to completion, +and it should return the result of executing the coroutine. Let's write a runner for ``trio`` that print a message when used as an exercise, ``trio`` is -special as it usually prefer to run a function object and make a coroutine by +special as it usually prefers to run a function object and make a coroutine by itself, we can get around this limitation by wrapping it in an async-def without parameters and passing this value to ``trio``:: @@ -246,8 +246,8 @@ Difference between terminal IPython and IPykernel The exact asynchronous code running behavior varies between Terminal IPython and IPykernel. The root cause of this behavior is due to IPykernel having a *persistent* `asyncio` loop running, while Terminal IPython starts and stops a -loop for each code block. This can lead to surprising behavior in some case if -you are used to manipulate asyncio loop yourself, see for example +loop for each code block. This can lead to surprising behavior in some cases if +you are used to manipulating asyncio loop yourself, see for example :ghissue:`11303` for a longer discussion but here are some of the astonishing cases. diff --git a/docs/source/interactive/magics.rst b/docs/source/interactive/magics.rst index bb562f44f20..b61bcdc7f85 100644 --- a/docs/source/interactive/magics.rst +++ b/docs/source/interactive/magics.rst @@ -13,9 +13,9 @@ Built-in magic commands other languages. Here is the help auto-generated from the docstrings of all the available Magics -function that IPython ships with. +functions that IPython ships with. -You can create an register your own Magics with IPython. You can find many user +You can create and register your own Magics with IPython. You can find many user defined Magics on `PyPI `_. Feel free to publish your own and use the ``Framework :: IPython`` trove classifier. diff --git a/docs/source/interactive/reference.rst b/docs/source/interactive/reference.rst index 08ac1867437..ccd37733252 100644 --- a/docs/source/interactive/reference.rst +++ b/docs/source/interactive/reference.rst @@ -77,8 +77,8 @@ prompt. What follows is a list of these. Caution for Windows users ------------------------- -Windows, unfortunately, uses the '\\' character as a path separator. This is a -terrible choice, because '\\' also represents the escape character in most +Windows, unfortunately, uses the ``\`` character as a path separator. This is a +terrible choice, because ``\`` also represents the escape character in most modern programming languages, including Python. For this reason, using '/' character is recommended if you have problems with ``\``. However, in Windows commands '/' flags options, so you can not use it for the root directory. This diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 9d4493a3a53..9e9b2984c6c 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -22,8 +22,12 @@ Need to be updated: pr/* + + .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. +As a reminder, IPython master has diverged from the 7.x branch, thus master may +have more feature and API changes. Backwards incompatible changes ------------------------------ diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 958954ebf5d..6bc36c0789c 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,190 @@ Issues closed in the 7.x development cycle ========================================== + +Issues closed in 7.16 +--------------------- + +GitHub stats for 2020/05/29 - 2020/06/26 (tag: 7.15.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 18 pull requests. +The full list can be seen `on GitHub `__ + +The following 7 authors contributed 22 commits. + +* Benjamin Ragan-Kelley +* dalthviz +* Frank Tobia +* Matthias Bussonnier +* palewire +* Paul McCarthy +* Talley Lambert + + +Issues closed in 7.15 +--------------------- + +GitHub stats for 2020/05/01 - 2020/05/29 (tag: 7.14.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 1 issues and merged 29 pull requests. +The full list can be seen `on GitHub `__ + +The following 6 authors contributed 31 commits. + +* Blake Griffin +* Inception95 +* Marcio Mazza +* Matthias Bussonnier +* Talley Lambert +* Thomas + +Issues closed in 7.14 +--------------------- + +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- + +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- + +GitHub stats for 2020/02/01 - 2020/02/28 (tag: 7.12.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 1 issues and merged 24 pull requests. +The full list can be seen `on GitHub `__ + +The following 12 authors contributed 108 commits. + +* Alex Hall +* Augusto +* Coon, Ethan T +* Daniel Hahler +* Inception95 +* Itamar Turner-Trauring +* Jonas Haag +* Jonathan Slenders +* linar-jether +* Matthias Bussonnier +* Nathan Goldbaum +* Terry Davis + + +Issues closed in 7.12 +--------------------- + +GitHub stats for 2020/01/01 - 2020/01/31 (tag: 7.11.1) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 2 issues and merged 14 pull requests. +The full list can be seen `on GitHub `__ + +The following 11 authors contributed 48 commits. + +* Augusto +* Eric Wieser +* Jeff Potter +* Mark E. Haase +* Matthias Bussonnier +* ossdev07 +* ras44 +* takuya fujiwara +* Terry Davis +* Thomas A Caswell +* yangyang + +Issues closed in 7.11 +--------------------- + +GitHub stats for 2019/12/01 - 2019/12/27 (tag: 7.10.1) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 4 issues and merged 36 pull requests. +The full list can be seen `on GitHub `__ + +The following 16 authors contributed 114 commits. + +* Augusto +* Benjamin Ragan-Kelley +* Chemss Eddine Ben Hassine +* Danny Hermes +* Dominik Miedziński +* Jonathan Feinberg +* Jonathan Slenders +* Joseph Kahn +* kousik +* Kousik Mitra +* Marc Hernandez Cabot +* Matthias Bussonnier +* Naveen Honest Raj K +* Pratyay Pandey +* Quentin Peter +* takuya fujiwara + + +Issues closed in 7.10.2 +----------------------- + + +GitHub stats for 2019/12/01 - 2019/12/14 (tag: 7.10.1) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 3 issues and merged 10 pull requests. +The full list can be seen `on GitHub `__ + +The following 3 authors contributed 11 commits. + +* Jonathan Slenders +* Joseph Kahn +* Matthias Bussonnier + Issues closed in 7.10.1 ----------------------- diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index c4ba18635b2..f8c88ab844b 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,311 @@ 7.x Series ============ +======= +.. _version 7.16.3: + +IPython 7.16.3 (CVE-2022-21699) +=============================== + +Fixed CVE-2022-21699, see IPython 8.0.1 release notes for informations. + +.. _version 716: + +IPython 7.16.1, 7.16.2 +====================== + +IPython 7.16.1 was release immediately after 7.16.0 to fix a conda packaging issue. +The source is identical to 7.16.0 but the file permissions in the tar are different. + +IPython 7.16.2 pins jedi dependency to "<=0.17.2" which should prevent some +issues for users still on python 3.6. This may not be sufficient as pip may +still allow to downgrade IPython. + +Compatibility with Jedi > 0.17.2 was not added as this would have meant bumping +the minimal version to >0.16. + +IPython 7.16 +============ + + +The default traceback mode will now skip frames that are marked with +``__tracebackhide__ = True`` and show how many traceback frames have been +skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or +``--hide`` attribute. It will have no effect on non verbose traceback modes. + +The ipython debugger also now understands ``__tracebackhide__`` as well and will +skip hidden frames when displaying. Movement up and down the stack will skip the +hidden frames and will show how many frames were hidden. Internal IPython frames +are also now hidden by default. The behavior can be changed with the +``skip_hidden`` while in the debugger, command and accepts "yes", "no", "true" +and "false" case insensitive parameters. + + +Misc Noticeable changes: +------------------------ + +- Exceptions are now (re)raised when running notebooks via the :magic:`%run`, helping to catch issues in workflows and + pipelines. :ghpull:`12301` +- Fix inputhook for qt 5.15.0 :ghpull:`12355` +- Fix wx inputhook :ghpull:`12375` +- Add handling for malformed pathext env var (Windows) :ghpull:`12367` +- use $SHELL in system_piped :ghpull:`12360` for uniform behavior with + ipykernel. + +Reproducible Build +------------------ + +IPython 7.15 reproducible build did not work, so we try again this month +:ghpull:`12358`. + + +API Changes +----------- + +Change of API and exposed objects automatically detected using `frappuccino +`_ (still in beta): + + +The following items are new and mostly related to understanding ``__tracebackbhide__``:: + + + IPython.core.debugger.Pdb.do_down(self, arg) + + IPython.core.debugger.Pdb.do_skip_hidden(self, arg) + + IPython.core.debugger.Pdb.do_up(self, arg) + + IPython.core.debugger.Pdb.hidden_frames(self, stack) + + IPython.core.debugger.Pdb.stop_here(self, frame) + + +The following items have been removed:: + + - IPython.core.debugger.Pdb.new_do_down + - IPython.core.debugger.Pdb.new_do_up + +Those were implementation details. + + +.. _version 715: + +IPython 7.15 +============ + +IPython 7.15 brings a number of bug fixes and user facing improvements. + +Misc Noticeable changes: +------------------------ + + - Long completion name have better elision in terminal :ghpull:`12284` + - I've started to test on Python 3.9 :ghpull:`12307` and fix some errors. + - Hi DPI scaling of figures when using qt eventloop :ghpull:`12314` + - Document the ability to have systemwide configuration for IPython. + :ghpull:`12328` + - Fix issues with input autoformatting :ghpull:`12336` + - ``IPython.core.debugger.Pdb`` is now interruptible (:ghpull:`12168`, in 7.14 + but forgotten in release notes) + - Video HTML attributes (:ghpull:`12212`, in 7.14 but forgotten in release + notes) + +Reproducible Build +------------------ + +Starting with IPython 7.15, I am attempting to provide reproducible builds, +that is to say you should be able from the source tree to generate an sdist +and wheel that are identical byte for byte with the publish version on PyPI. + +I've only tested on a couple of machines so far and the process is relatively +straightforward, so this mean that IPython not only have a deterministic build +process, but also I have either removed, or put under control all effects of +the build environments on the final artifact. I encourage you to attempt the +build process on your machine as documented in :ref:`core_developer_guide` +and let me know if you do not obtain an identical artifact. + +While reproducible builds is critical to check that the supply chain of (open +source) software has not been compromised, it can also help to speedup many +of the build processes in large environment (conda, apt...) by allowing +better caching of intermediate build steps. + +Learn more on ``_. `Reflections on trusting +trust `_ is also one of the +cornerstone and recommended reads on this subject. + +.. note:: + + The build commit from which the sdist is generated is also `signed + `_, so you should be able to + check it has not been compromised, and the git repository is a `merkle-tree + `_, you can check the consistency + with `git-fsck `_ which you likely `want + to enable by default + `_. + +NEP29: Last version to support Python 3.6 +----------------------------------------- + +IPython 7.15 will be the Last IPython version to officially support Python +3.6, as stated by `NumPy Enhancement Proposal 29 +`_. Starting with +next minor version of IPython I may stop testing on Python 3.6 and may stop +publishing release artifacts that install on Python 3.6 + +Highlighted features +-------------------- + +Highlighted features are not new, but seem to not be widely known, this +section will help you discover in more narrative form what you can do with +IPython. + +Increase Tab Completion Menu Height +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In terminal IPython it is possible to increase the hight of the tab-completion +menu. To do so set the value of +:configtrait:`TerminalInteractiveShell.space_for_menu`, this will reserve more +space at the bottom of the screen for various kind of menus in IPython including +tab completion and searching in history. + +Autoformat Code in the terminal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a preferred code formatter, you can configure IPython to +reformat your code. Set the value of +:configtrait:`TerminalInteractiveShell.autoformatter` to for example ``'black'`` +and IPython will auto format your code when possible. + + +.. _version 714: + +IPython 7.14 +============ + +IPython 7.14 is a minor release that fix a couple of bugs and prepare +compatibility with new or future versions of some libraries. + +Important changes: +------------------ + + - Fix compatibility with Sphinx 3+ :ghpull:`12235` + - Remove deprecated matplotlib parameter usage, compatibility with matplotlib + 3.3+ :`122250` + +Misc Changes +------------ + + - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` + - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` + - add extra options to the ``Video`` Rich objects :ghpull:`12212` + - add pretty-printing to ``SimpleNamespace`` :ghpull:`12230` + +IPython.core.debugger.Pdb is now interruptible +---------------------------------------------- + +A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. (:ghpull:`12168`) + +Video HTML attributes +--------------------- + +Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video (:ghpull:`12212`) + + +Pending deprecated imports +-------------------------- + +Many object present in ``IPython.core.display`` are there for internal use only, +and should already been imported from ``IPython.display`` by users and external +libraries. Trying to import those from ``IPython.core.display`` is still possible +but will trigger a +deprecation warning in later versions of IPython and will become errors in the +future. + +This will simplify compatibility with other Python kernels (like Xeus-Python), +and simplify code base. + + + + +.. _version 713: + +IPython 7.13 +============ + +IPython 7.13 is the final release of the 7.x branch since master is diverging +toward an 8.0. Exiting new features have already been merged in 8.0 and will +not be available on the 7.x branch. All the changes below have been backported +from the master branch. + + + - Fix inability to run PDB when inside an event loop :ghpull:`12141` + - Fix ability to interrupt some processes on windows :ghpull:`12137` + - Fix debugger shortcuts :ghpull:`12132` + - improve tab completion when inside a string by removing irrelevant elements :ghpull:`12128` + - Fix display of filename tab completion when the path is long :ghpull:`12122` + - Many removal of Python 2 specific code path :ghpull:`12110` + - displaying wav files do not require NumPy anymore, and is 5x to 30x faster :ghpull:`12113` + +See the list of all closed issues and pull request on `github +`_. + +.. _version 712: + +IPython 7.12 +============ + +IPython 7.12 is a minor update that mostly brings code cleanup, removal of +longtime deprecated function and a couple update to documentation cleanup as well. + +Notable changes are the following: + + - Exit non-zero when ipython is given a file path to run that doesn't exist :ghpull:`12074` + - Test PR on ARM64 with Travis-CI :ghpull:`12073` + - Update CI to work with latest Pytest :ghpull:`12086` + - Add infrastructure to run ipykernel eventloop via trio :ghpull:`12097` + - Support git blame ignore revs :ghpull:`12091` + - Start multi-line ``__repr__`` s on their own line :ghpull:`12099` + +.. _version 7111: + +IPython 7.11.1 +============== + +A couple of deprecated functions (no-op) have been reintroduces in py3compat as +Cython was still relying on them, and will be removed in a couple of versions. + +.. _version 711: + +IPython 7.11 +============ + +IPython 7.11 received a couple of compatibility fixes and code cleanup. + +A number of function in the ``py3compat`` have been removed; a number of types +in the IPython code base are now non-ambiguous and now always ``unicode`` +instead of ``Union[Unicode,bytes]``; many of the relevant code path have thus +been simplified/cleaned and types annotation added. + +IPython support several verbosity level from exceptions. ``xmode plain`` now +support chained exceptions. :ghpull:`11999` + +We are starting to remove ``shell=True`` in some usages of subprocess. While not directly +a security issue (as IPython is made to run arbitrary code anyway) it is not good +practice and we'd like to show the example. :ghissue:`12023`. This discussion +was started by ``@mschwager`` thanks to a new auditing tool they are working on +with duo-labs (`dlint `_). + +Work around some bugs in Python 3.9 tokenizer :ghpull:`12057` + +IPython will now print its version after a crash. :ghpull:`11986` + +This is likely the last release from the 7.x series that will see new feature. +The master branch will soon accept large code changes and thrilling new +features; the 7.x branch will only start to accept critical bug fixes, and +update dependencies. + +.. _version 7102: + +IPython 7.10.2 +============== + +IPython 7.10.2 fix a couple of extra incompatibility between IPython, ipdb, +asyncio and Prompt Toolkit 3. + .. _version 7101: IPython 7.10.1 diff --git a/docs/sphinxext/configtraits.py b/docs/sphinxext/configtraits.py index 4e767694097..2b05d2bf6c8 100644 --- a/docs/sphinxext/configtraits.py +++ b/docs/sphinxext/configtraits.py @@ -8,8 +8,7 @@ Cross reference like this: :configtrait:`Application.log_datefmt`. """ -from sphinx.locale import l_ -from sphinx.util.docfields import Field + def setup(app): app.add_object_type('configtrait', 'configtrait', objname='Config option') diff --git a/examples/IPython Kernel/ipython-get-history.py b/examples/IPython Kernel/ipython-get-history.py index 561584253a9..5e68bf5c041 100755 --- a/examples/IPython Kernel/ipython-get-history.py +++ b/examples/IPython Kernel/ipython-get-history.py @@ -35,5 +35,4 @@ hist = HistoryAccessor() for session, lineno, cell in hist.get_range(session=session_number, raw=raw): - cell = cell.encode('utf-8') # This line is only needed on Python 2. dest.write(cell + '\n') diff --git a/pytest.ini b/pytest.ini index 537ebeb2674..9da365334c6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --duration=10 +addopts = --durations=10 diff --git a/setup.py b/setup.py index 593e3a6a0af..3e734bafe14 100755 --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ install_requires = [ 'setuptools>=18.5', - 'jedi>=0.10', + 'jedi>=0.10,<=0.17.2', 'decorator', 'pickleshare', 'traitlets>=4.2', @@ -225,7 +225,7 @@ for key, deps in extras_require.items(): if ':' not in key: everything.update(deps) -extras_require['all'] = everything +extras_require['all'] = list(sorted(everything)) if 'setuptools' in sys.modules: setuptools_extra_args['python_requires'] = '>=3.6' diff --git a/tools/autoformat_file b/tools/autoformat_file new file mode 100755 index 00000000000..3b6b71d3007 --- /dev/null +++ b/tools/autoformat_file @@ -0,0 +1,46 @@ +#!/bin/bash + +set -ueo pipefail +FILE=$1 + +echo "will update $FILE" + +echo $LINENO $? +pyupgrade --py36-plus --exit-zero-even-if-changed $FILE + +echo $LINENO $? +git commit -am"Apply pyupgrade to $FILE + + pyupgrade --py36-plus $FILE + +To ignore those changes when using git blame see the content of +.git-blame-ignore-revs" + +HASH=$(git rev-parse HEAD) + +echo "$HASH # apply pyupgrade to $FILE" >> .git-blame-ignore-revs + +git commit -am'Update .git-blame-ignore-revs with previous commit' + +##### + +black --target-version py36 $FILE + + +git commit -am"Apply black to $FILE + + black --target-version py36 $FILE + +To ignore those changes when using git blame see the content of +.git-blame-ignore-revs" + +HASH=$(git rev-parse HEAD) + +echo "$HASH # apply black to $FILE" >> .git-blame-ignore-revs + +git commit -am'Update .git-blame-ignore-revs with previous commit' + +echo +echo "Updating, reformatting and adding to .git-blame-ignore-revs successful" + + diff --git a/tools/build_release b/tools/build_release index 26dc9ec874a..51fd87d54d9 100755 --- a/tools/build_release +++ b/tools/build_release @@ -2,6 +2,7 @@ """IPython release build script. """ import os +import sys from shutil import rmtree from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels @@ -12,15 +13,10 @@ def build_release(): ipdir = get_ipdir() cd(ipdir) - # Cleanup - for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'), - pjoin('docs', 'source', 'api', 'generated')]: - if os.path.isdir(d): - rmtree(d) - # Build source and binary distros sh(sdists) buildwheels() + sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz'])) if __name__ == '__main__': build_release() diff --git a/tools/configure-git-blame-ignore-revs.bat b/tools/configure-git-blame-ignore-revs.bat new file mode 100644 index 00000000000..3fc3c01dcf8 --- /dev/null +++ b/tools/configure-git-blame-ignore-revs.bat @@ -0,0 +1,10 @@ +rem Other config options for blame are markUnblamables and markIgnoredLines. +rem See docs for more details: +rem https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +rem Uncomment below and rerun script to enable an option. +rem git config blame.markIgnoredLines +rem git config blame.markUnblamables + +git config blame.ignoreRevsFile .git-blame-ignore-revs +git config --get blame.ignoreRevsFile diff --git a/tools/configure-git-blame-ignore-revs.sh b/tools/configure-git-blame-ignore-revs.sh new file mode 100644 index 00000000000..e1892ef4d05 --- /dev/null +++ b/tools/configure-git-blame-ignore-revs.sh @@ -0,0 +1,10 @@ +# Other config options for blame are markUnblamables and markIgnoredLines. +# See docs for more details: +# https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +# Uncomment below and rerun script to enable an option. +# git config blame.markIgnoredLines +# git config blame.markUnblamables + +git config blame.ignoreRevsFile .git-blame-ignore-revs +git config --get blame.ignoreRevsFile diff --git a/tools/make_tarball.py b/tools/make_tarball.py index bdce25ba804..fb639f61f61 100755 --- a/tools/make_tarball.py +++ b/tools/make_tarball.py @@ -3,7 +3,6 @@ """ import subprocess -import os from toollib import cd, sh diff --git a/tools/release b/tools/release index 5c8686b3145..2de8e120070 100755 --- a/tools/release +++ b/tools/release @@ -81,13 +81,10 @@ else: sh('mv ipython-*.tgz %s' % ipbackupdir) # Build release files - sh('./build_release %s' % ipdir) + sh('./build_release') cd(ipdir) - # Upload all files - sh(sdists) - buildwheels() print("`./release upload` to upload source distribution on PyPI and ipython archive") sys.exit(0) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index bed8ed69253..cf053d3459a 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -1,5 +1,5 @@ # Simple tool to help for release -# when releasing with bash, simplei source it to get asked questions. +# when releasing with bash, simple source it to get asked questions. # misc check before starting @@ -7,7 +7,7 @@ python -c 'import keyring' python -c 'import twine' python -c 'import sphinx' python -c 'import sphinx_rtd_theme' -python -c 'import nose' +python -c 'import pytest' BLACK=$(tput setaf 1) @@ -21,6 +21,7 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) +echo "Will use '$EDITOR' to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input PREV_RELEASE=${input:-$PREV_RELEASE} @@ -30,26 +31,48 @@ MILESTONE=${input:-$MILESTONE} echo -n "VERSION (X.y.z) [$VERSION]:" read input VERSION=${input:-$VERSION} -echo -n "branch (master|X.y) [$branch]:" +echo -n "BRANCH (master|X.y) [$BRANCH]:" read input -branch=${input:-$branch} +BRANCH=${input:-$BRANCH} ask_section(){ echo echo $BLUE"$1"$NOR - echo -n $GREEN"Press Enter to continue, S to skip: "$GREEN - read -n1 value - echo - if [ -z $value ] || [ $value = 'y' ] ; then + echo -n $GREEN"Press Enter to continue, S to skip: "$NOR + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + if [ -z "$value" ] || [ $value = 'y' ]; then return 0 fi return 1 } +maybe_edit(){ + echo + echo $BLUE"$1"$NOR + echo -n $GREEN"Press e to Edit $1, any other keys to skip: "$NOR + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + + echo + if [ $value = 'e' ] ; then + $=EDITOR $1 + fi +} + + echo -if ask_section "Updating what's new with informations from docs/source/whatsnew/pr" +if ask_section "Updating what's new with information from docs/source/whatsnew/pr" then python tools/update_whatsnew.py @@ -81,56 +104,146 @@ then fi +if ask_section "Generate API difference (using frapuccino)" +then + echo $BLUE"Checking out $PREV_RELEASE"$NOR + git checkout $PREV_RELEASE + echo $BLUE"Saving API to file $PREV_RELEASE"$NOR + frappuccino IPython --save IPython-$PREV_RELEASE.json + echo $BLUE"comming back to $BRANCH"$NOR + git checkout $BRANCH + echo $BLUE"comparing ..."$NOR + frappuccino IPython --compare IPython-$PREV_RELEASE.json + echo $GREEN"Use the above guideline to write an API changelog ..."$NOR + echo $GREEN"Press any keys to continue"$NOR + read +fi + echo "Cleaning repository" git clean -xfdi echo $GREEN"please update version number in ${RED}IPython/core/release.py${NOR} , Do not commit yet – we'll do it later."$NOR +echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py${NOR}" +sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py +rm IPython/core/release.pybkp +git diff | cat +maybe_edit IPython/core/release.py echo $GREEN"Press enter to continue"$NOR read -echo -echo "Attempting to build the docs.." -make html -C docs +if ask_section "Build the documentation ?" +then + make html -C docs + echo + echo $GREEN"Check the docs, press enter to continue"$NOR + read -echo -echo $GREEN"Check the docs, press enter to continue"$NOR -read +fi -echo -echo $BLUE"Attempting to build package..."$NOR +if ask_section "Should we commit, tag, push... etc ? " +then + echo + echo $BLUE"Let's commit : git commit -am \"release $VERSION\" -S" + echo $GREEN"Press enter to commit"$NOR + read + git commit -am "release $VERSION" -S + + echo + echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR + echo $GREEN"Make sure you can push"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git push origin $BRANCH + + echo + echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s" + echo $GREEN"Press enter to tag commit"$NOR + read + git tag -am "release $VERSION" "$VERSION" -s + + echo + echo $BLUE"And push the tag: git push origin \$VERSION ?"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git push origin $VERSION + + + echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py" + echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py${NOR}" + sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py + rm IPython/core/release.pybkp + git diff | cat + echo $GREEN"Please bump ${RED}the minor version number${NOR}" + maybe_edit IPython/core/release.py + echo ${BLUE}"Do not commit yet – we'll do it later."$NOR + + + echo $GREEN"Press enter to continue"$NOR + read + + echo + echo "Let's commit : "$BLUE"git commit -am \"back to dev\""$NOR + echo $GREEN"Press enter to commit"$NOR + read + git commit -am "back to dev" + + echo + echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git push origin $BRANCH + + + echo + echo $BLUE"let's : git checkout $VERSION"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git checkout $VERSION +fi -tools/build_release +if ask_section "Should we build and release ?" +then + + echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR + echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR + echo $GREEN"Press enter to continue"$NOR + read -echo -echo "Let's commit : git commit -am \"release $VERSION\" -S" -echo $GREEN"Press enter to commit"$NOR -read -git commit -am "release $VERSION" -S + export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) -echo -echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR -echo $GREEN"Make sure you can push"$NOR -echo $GREEN"Press enter to continue"$NOR -read -git push origin $BRANCH + echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR + echo $GREEN"Press enter to continue"$NOR + read -echo -echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s" -echo $GREEN"Press enter to wtagcommit"$NOR -read -git tag -am "release $VERSION" "$VERSION" -s -echo -echo $BLUE"And push the tag: git push origin \$VERSION ?"$NOR -echo $GREEN"Press enter to continue"$NOR -read -git push origin $VERSION -echo -echo $BLUE"let's : git checkout $VERSION"$NOR -echo $GREEN"Press enter to continue"$NOR -read -git checkout $VERSION + echo + echo $BLUE"Attempting to build package..."$NOR + + tools/release + + + echo $RED'$ shasum -a 256 dist/*' + shasum -a 256 dist/* + echo $NOR + + echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR + echo $GREEN"Press enter to continue"$NOR + read + + echo + echo $BLUE"Attempting to build package..."$NOR + tools/release + echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + echo $RED'$ shasum -a 256 dist/*' + shasum -a 256 dist/* + echo $NOR + + if ask_section "upload packages ?" + then + tools/release upload + fi +fi diff --git a/tools/retar.py b/tools/retar.py new file mode 100644 index 00000000000..efd8c0c6bbd --- /dev/null +++ b/tools/retar.py @@ -0,0 +1,65 @@ +""" +Un-targz and retargz a targz file to ensure reproducible build. + +usage: + + $ export SOURCE_DATE_EPOCH=$(date +%s) + ... + $ python retar.py + +The process of creating an sdist can be non-reproducible: + - directory created during the process get a mtime of the creation date; + - gziping files embed the timestamp of fo zip creation. + +This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set +equal to SOURCE_DATE_EPOCH. + +""" + +import tarfile +import sys +import os +import gzip +import io + +if len(sys.argv) > 2: + raise ValueError("Too many arguments") + + +timestamp = int(os.environ["SOURCE_DATE_EPOCH"]) + +old_buf = io.BytesIO() +with open(sys.argv[1], "rb") as f: + old_buf.write(f.read()) +old_buf.seek(0) +old = tarfile.open(fileobj=old_buf, mode="r:gz") + +buf = io.BytesIO() +new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT) +for i, m in enumerate(old): + data = None + # mutation does not work, copy + if m.name.endswith('.DS_Store'): + continue + m2 = tarfile.TarInfo(m.name) + m2.mtime = min(timestamp, m.mtime) + m2.size = m.size + m2.type = m.type + m2.linkname = m.linkname + m2.mode = m.mode + if m.isdir(): + new.addfile(m2) + else: + data = old.extractfile(m) + new.addfile(m2, data) +new.close() +old.close() + +buf.seek(0) +with open(sys.argv[1], "wb") as f: + with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: + gzf.write(buf.read()) + +# checks the archive is valid. +archive = tarfile.open(sys.argv[1]) +names = archive.getnames()