diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 73cd8d31d0b20d..d0355ce47a6504 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -1338,6 +1338,13 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_COMPLETER + + If this variable is set to any value, PyREPL will use :mod:`rlcompleter` to + implement tab completion, instead of the default one which uses colors. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 0b23bb48dd5c1b..d4dbe541e6da51 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -691,7 +691,11 @@ struct _obmalloc_state { /* Allocate memory directly from the O/S virtual memory system, - * where supported. Otherwise fallback on malloc */ + * where supported. Otherwise fallback on malloc. + * + * Large-page and huge-page backends may round the mapped size up + * internally, so pass the original requested size back to + * _PyObject_VirtualFree(). */ void *_PyObject_VirtualAlloc(size_t size); void _PyObject_VirtualFree(void *, size_t size); diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 75acdf596a7f30..7fc41a807dd566 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -507,6 +507,38 @@ After the last `DEOPT_IF` has passed, a hit should be recorded with After an optimization has been deferred in the adaptive instruction, that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. +## Interpreter types +There are three different types of interpreters to choose from based on compiler support: + + * traditional switch-case interpreter + + Supported by all compilers covered in PEP 7. + + * computed-gotos interpreter + + Enabled using configure option `--with-computed-gotos` and used by default on supported compilers. + It uses [Labels as Values](https://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html) + for more efficient dispatching. + + * tail-calling interpreter + + Enabled using configure option `--with-tail-call-interp` (or `--tail-call-interp` for build.bat on Windows). + It uses [tail calls](https://clang.llvm.org/docs/AttributeReference.html#musttail) and the + [preserve_none](https://clang.llvm.org/docs/AttributeReference.html#preserve-none) + calling convention between the small C functions that implement individual Python opcodes. + + Not all compilers support these and if they do not all targets might be supported (for example, + MSVC currently only supports x64 and only in optimized builds). + + In addition, compilers must do [escape analysis](https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-musttail) + of the lifetimes of automatic variables, function parameters, and temporaries to ensure proper tail-calls. They + emit a compile error in case of a violation or detection failure. The ability to detect this varies depending on the compiler and + also on the optimization level. Following techniques are particularly helpful to the MSVC compiler in this regard + * [Introducing additional scopes](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Python/bytecodes.c#L2526) + * [extracting problematic code paths into a separate function](https://github.com/python/cpython/pull/143068/files#diff-729a985b0cb8b431cb291f1edb561bbbfea22e3f8c262451cd83328a0936a342R3724) + * [returning a pointer instead of taking it as an output parameter](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Include/internal/pycore_ceval.h#L489-L492) + + Using `restrict` is another (currently unused) remedy. Additional resources -------------------- diff --git a/Lib/_colorize.py b/Lib/_colorize.py index fd0ae9d6145961..8361ddbea89716 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -15,7 +15,6 @@ class ANSIColors: RESET = "\x1b[0m" - BLACK = "\x1b[30m" BLUE = "\x1b[34m" CYAN = "\x1b[36m" @@ -200,6 +199,30 @@ class Difflib(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class FancyCompleter(ThemeSection): + # functions and methods + function: str = ANSIColors.BOLD_BLUE + builtin_function_or_method: str = ANSIColors.BOLD_BLUE + method: str = ANSIColors.BOLD_CYAN + method_wrapper: str = ANSIColors.BOLD_CYAN + wrapper_descriptor: str = ANSIColors.BOLD_CYAN + method_descriptor: str = ANSIColors.BOLD_CYAN + + # numbers + int: str = ANSIColors.BOLD_YELLOW + float: str = ANSIColors.BOLD_YELLOW + complex: str = ANSIColors.BOLD_YELLOW + bool: str = ANSIColors.BOLD_YELLOW + + # others + type: str = ANSIColors.BOLD_MAGENTA + module: str = ANSIColors.CYAN + NoneType: str = ANSIColors.GREY + bytes: str = ANSIColors.BOLD_GREEN + str: str = ANSIColors.BOLD_GREEN + + @dataclass(frozen=True, kw_only=True) class LiveProfiler(ThemeSection): """Theme section for the live profiling TUI (Tachyon profiler). @@ -354,6 +377,7 @@ class Theme: """ argparse: Argparse = field(default_factory=Argparse) difflib: Difflib = field(default_factory=Difflib) + fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) live_profiler: LiveProfiler = field(default_factory=LiveProfiler) syntax: Syntax = field(default_factory=Syntax) traceback: Traceback = field(default_factory=Traceback) @@ -364,6 +388,7 @@ def copy_with( *, argparse: Argparse | None = None, difflib: Difflib | None = None, + fancycompleter: FancyCompleter | None = None, live_profiler: LiveProfiler | None = None, syntax: Syntax | None = None, traceback: Traceback | None = None, @@ -377,6 +402,7 @@ def copy_with( return type(self)( argparse=argparse or self.argparse, difflib=difflib or self.difflib, + fancycompleter=fancycompleter or self.fancycompleter, live_profiler=live_profiler or self.live_profiler, syntax=syntax or self.syntax, traceback=traceback or self.traceback, @@ -394,6 +420,7 @@ def no_colors(cls) -> Self: return cls( argparse=Argparse.no_colors(), difflib=Difflib.no_colors(), + fancycompleter=FancyCompleter.no_colors(), live_profiler=LiveProfiler.no_colors(), syntax=Syntax.no_colors(), traceback=Traceback.no_colors(), diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 9d2d43be5144e8..5802920a907ca4 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -178,12 +178,14 @@ def do(self) -> None: if not completions: r.error("no matches") elif len(completions) == 1: - if completions_unchangable and len(completions[0]) == len(stem): + completion = stripcolor(completions[0]) + if completions_unchangable and len(completion) == len(stem): r.msg = "[ sole completion ]" r.dirty = True - r.insert(completions[0][len(stem):]) + r.insert(completion[len(stem):]) else: - p = prefix(completions, len(stem)) + clean_completions = [stripcolor(word) for word in completions] + p = prefix(clean_completions, len(stem)) if p: r.insert(p) if last_is_completer: @@ -195,7 +197,7 @@ def do(self) -> None: r.dirty = True elif not r.cmpltn_menu_visible: r.cmpltn_message_visible = True - if stem + p in completions: + if stem + p in clean_completions: r.msg = "[ complete but not unique ]" r.dirty = True else: @@ -215,7 +217,7 @@ def do(self) -> None: r.cmpltn_reset() else: completions = [w for w in r.cmpltn_menu_choices - if w.startswith(stem)] + if stripcolor(w).startswith(stem)] if completions: r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, 0, diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py new file mode 100644 index 00000000000000..5b5b7ae5f2bb59 --- /dev/null +++ b/Lib/_pyrepl/fancycompleter.py @@ -0,0 +1,210 @@ +# Copyright 2010-2025 Antonio Cuni +# Daniel Hahler +# +# All Rights Reserved +"""Colorful tab completion for Python prompt""" +from _colorize import ANSIColors, get_colors, get_theme +import rlcompleter +import keyword +import types + +class Completer(rlcompleter.Completer): + """ + When doing something like a.b., keep the full a.b.attr completion + stem so readline-style completion can keep refining the menu as you type. + + Optionally, display the various completions in different colors + depending on the type. + """ + def __init__( + self, + namespace=None, + *, + use_colors='auto', + consider_getitems=True, + ): + from _pyrepl import readline + rlcompleter.Completer.__init__(self, namespace) + if use_colors == 'auto': + # use colors only if we can + use_colors = get_colors().RED != "" + self.use_colors = use_colors + self.consider_getitems = consider_getitems + + if self.use_colors: + # In GNU readline, this prevents escaping of ANSI control + # characters in completion results. pyrepl's parse_and_bind() + # is a no-op, but pyrepl handles ANSI sequences natively + # via real_len()/stripcolor(). + readline.parse_and_bind('set dont-escape-ctrl-chars on') + self.theme = get_theme() + else: + self.theme = None + + if self.consider_getitems: + delims = readline.get_completer_delims() + delims = delims.replace('[', '') + delims = delims.replace(']', '') + readline.set_completer_delims(delims) + + def complete(self, text, state): + # if you press at the beginning of a line, insert an actual + # \t. Else, trigger completion. + if text == "": + return ('\t', None)[state] + else: + return rlcompleter.Completer.complete(self, text, state) + + def _callable_postfix(self, val, word): + # disable automatic insertion of '(' for global callables + return word + + def _callable_attr_postfix(self, val, word): + return rlcompleter.Completer._callable_postfix(self, val, word) + + def global_matches(self, text): + names = rlcompleter.Completer.global_matches(self, text) + prefix = commonprefix(names) + if prefix and prefix != text: + return [prefix] + + names.sort() + values = [] + for name in names: + clean_name = name.rstrip(': ') + if keyword.iskeyword(clean_name) or keyword.issoftkeyword(clean_name): + values.append(None) + else: + try: + values.append(eval(name, self.namespace)) + except Exception: + values.append(None) + if self.use_colors and names: + return self.colorize_matches(names, values) + return names + + def attr_matches(self, text): + try: + expr, attr, names, values = self._attr_matches(text) + except ValueError: + return [] + + if not names: + return [] + + if len(names) == 1: + # No coloring: when returning a single completion, readline + # inserts it directly into the prompt, so ANSI codes would + # appear as literal characters. + return [self._callable_attr_postfix(values[0], f'{expr}.{names[0]}')] + + prefix = commonprefix(names) + if prefix and prefix != attr: + return [f'{expr}.{prefix}'] # autocomplete prefix + + names = [f'{expr}.{name}' for name in names] + if self.use_colors: + return self.colorize_matches(names, values) + + if prefix: + names.append(' ') + return names + + def _attr_matches(self, text): + expr, attr = text.rsplit('.', 1) + if '(' in expr or ')' in expr: # don't call functions + return expr, attr, [], [] + try: + thisobject = eval(expr, self.namespace) + except Exception: + return expr, attr, [], [] + + # get the content of the object, except __builtins__ + words = set(dir(thisobject)) - {'__builtins__'} + + if hasattr(thisobject, '__class__'): + words.add('__class__') + words.update(rlcompleter.get_class_members(thisobject.__class__)) + names = [] + values = [] + n = len(attr) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + + # sort the words now to make sure to return completions in + # alphabetical order. It's easier to do it now, else we would need to + # sort 'names' later but make sure that 'values' in kept in sync, + # which is annoying. + words = sorted(words) + while True: + for word in words: + if ( + word[:n] == attr + and not (noprefix and word[:n+1] == noprefix) + ): + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(thisobject), word, None), property): + value = None + elif ( + isinstance(thisobject, types.ModuleType) + and isinstance( + thisobject.__dict__.get(word), + types.LazyImportType, + ) + ): + value = thisobject.__dict__.get(word) + else: + value = getattr(thisobject, word, None) + + names.append(word) + values.append(value) + if names or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + + return expr, attr, names, values + + def colorize_matches(self, names, values): + matches = [self._color_for_obj(i, name, obj) + for i, (name, obj) + in enumerate(zip(names, values))] + # We add a space at the end to prevent the automatic completion of the + # common prefix, which is the ANSI escape sequence. + matches.append(' ') + return matches + + def _color_for_obj(self, i, name, value): + t = type(value) + color = self._color_by_type(t) + # Encode the match index into a fake escape sequence that + # stripcolor() can still remove once i reaches four digits. + N = f"\x1b[{i // 100:03d};{i % 100:02d}m" + return f"{N}{color}{name}{ANSIColors.RESET}" + + def _color_by_type(self, t): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(self.theme.fancycompleter, typename, ANSIColors.RESET) + + +def commonprefix(names): + """Return the common prefix of all 'names'""" + if not names: + return '' + s1 = min(names) + s2 = max(names) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 23b8fa6b9c7625..17319963b1950a 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -40,6 +40,7 @@ from .completing_reader import CompletingReader from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer +from .fancycompleter import Completer as FancyCompleter Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -609,7 +610,12 @@ def _setup(namespace: Mapping[str, Any]) -> None: if not isinstance(namespace, dict): namespace = dict(namespace) _wrapper.config.module_completer = ModuleCompleter(namespace) - _wrapper.config.readline_completer = RLCompleter(namespace).complete + use_basic_completer = ( + not sys.flags.ignore_environment + and os.getenv("PYTHON_BASIC_COMPLETER") + ) + completer_cls = RLCompleter if use_basic_completer else FancyCompleter + _wrapper.config.readline_completer = completer_cls(namespace).complete # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/test/test_pyrepl/test_fancycompleter.py b/Lib/test/test_pyrepl/test_fancycompleter.py new file mode 100644 index 00000000000000..77c80853a3c0e3 --- /dev/null +++ b/Lib/test/test_pyrepl/test_fancycompleter.py @@ -0,0 +1,247 @@ +import importlib +import os +import types +import unittest + +from _colorize import ANSIColors, get_theme +from _pyrepl.completing_reader import stripcolor +from _pyrepl.fancycompleter import Completer, commonprefix +from test.support.import_helper import ready_to_import + +class MockPatch: + def __init__(self): + self.original_values = {} + + def setattr(self, obj, name, value): + if obj not in self.original_values: + self.original_values[obj] = {} + if name not in self.original_values[obj]: + self.original_values[obj][name] = getattr(obj, name) + setattr(obj, name, value) + + def restore_all(self): + for obj, attrs in self.original_values.items(): + for name, value in attrs.items(): + setattr(obj, name, value) + +class FancyCompleterTests(unittest.TestCase): + def setUp(self): + self.mock_patch = MockPatch() + + def tearDown(self): + self.mock_patch.restore_all() + + def test_commonprefix(self): + self.assertEqual(commonprefix(['isalpha', 'isdigit', 'foo']), '') + self.assertEqual(commonprefix(['isalpha', 'isdigit']), 'is') + self.assertEqual(commonprefix([]), '') + + def test_complete_attribute(self): + compl = Completer({'a': None}, use_colors=False) + self.assertEqual(compl.attr_matches('a.'), ['a.__']) + matches = compl.attr_matches('a.__') + self.assertNotIn('__class__', matches) + self.assertIn('a.__class__', matches) + match = compl.attr_matches('a.__class') + self.assertEqual(len(match), 1) + self.assertTrue(match[0].startswith('a.__class__')) + + def test_complete_attribute_prefix(self): + class C(object): + attr = 1 + _attr = 2 + __attr__attr = 3 + compl = Completer({'a': C}, use_colors=False) + self.assertEqual(compl.attr_matches('a.'), ['a.attr', 'a.mro']) + self.assertEqual( + compl.attr_matches('a._'), + ['a._C__attr__attr', 'a._attr', ' '], + ) + matches = compl.attr_matches('a.__') + self.assertNotIn('__class__', matches) + self.assertIn('a.__class__', matches) + match = compl.attr_matches('a.__class') + self.assertEqual(len(match), 1) + self.assertTrue(match[0].startswith('a.__class__')) + + compl = Completer({'a': None}, use_colors=False) + self.assertEqual(compl.attr_matches('a._'), ['a.__']) + + def test_complete_attribute_colored(self): + theme = get_theme() + compl = Completer({'a': 42}, use_colors=True) + matches = compl.attr_matches('a.__') + self.assertGreater(len(matches), 2) + expected_color = theme.fancycompleter.type + expected_part = f'{expected_color}a.__class__{ANSIColors.RESET}' + for match in matches: + if expected_part in match: + break + else: + self.assertFalse(True, matches) + self.assertIn(' ', matches) + + def test_preserves_callable_postfix_for_single_attribute_match(self): + compl = Completer({'os': os}, use_colors=False) + self.assertEqual(compl.attr_matches('os.getpid'), ['os.getpid()']) + + def test_property_method_not_called(self): + class Foo: + property_called = False + + @property + def bar(self): + self.property_called = True + return 1 + + foo = Foo() + compl = Completer({'foo': foo}, use_colors=False) + self.assertEqual(compl.attr_matches('foo.b'), ['foo.bar']) + self.assertFalse(foo.property_called) + + def test_excessive_getattr(self): + class Foo: + calls = 0 + bar = '' + + def __getattribute__(self, name): + if name == 'bar': + self.calls += 1 + return None + return super().__getattribute__(name) + + foo = Foo() + compl = Completer({'foo': foo}, use_colors=False) + self.assertEqual(compl.complete('foo.b', 0), 'foo.bar') + self.assertEqual(foo.calls, 1) + + def test_uncreated_attr(self): + class Foo: + __slots__ = ('bar',) + + compl = Completer({'foo': Foo()}, use_colors=False) + self.assertEqual(compl.complete('foo.', 0), 'foo.bar') + + def test_module_attributes_do_not_reify_lazy_imports(self): + with ready_to_import("test_pyrepl_lazy_mod", "lazy import json\n") as (name, _): + module = importlib.import_module(name) + self.assertIs(type(module.__dict__["json"]), types.LazyImportType) + + compl = Completer({name: module}, use_colors=False) + self.assertEqual(compl.attr_matches(f"{name}.j"), [f"{name}.json"]) + self.assertIs(type(module.__dict__["json"]), types.LazyImportType) + + def test_complete_colored_single_match(self): + """No coloring, via commonprefix.""" + compl = Completer({'foobar': 42}, use_colors=True) + matches = compl.global_matches('foob') + self.assertEqual(matches, ['foobar']) + + def test_does_not_color_single_match(self): + class obj: + msgs = [] + + compl = Completer({'obj': obj}, use_colors=True) + matches = compl.attr_matches('obj.msgs') + self.assertEqual(matches, ['obj.msgs']) + + def test_complete_global(self): + compl = Completer({'foobar': 1, 'foobazzz': 2}, use_colors=False) + self.assertEqual(compl.global_matches('foo'), ['fooba']) + matches = compl.global_matches('fooba') + self.assertEqual(set(matches), set(['foobar', 'foobazzz'])) + self.assertEqual(compl.global_matches('foobaz'), ['foobazzz']) + self.assertEqual(compl.global_matches('nothing'), []) + + def test_complete_global_colored(self): + theme = get_theme() + compl = Completer({'foobar': 1, 'foobazzz': 2}, use_colors=True) + self.assertEqual(compl.global_matches('foo'), ['fooba']) + matches = compl.global_matches('fooba') + + # these are the fake escape sequences which are needed so that + # readline displays the matches in the proper order + N0 = f"\x1b[000;00m" + N1 = f"\x1b[000;01m" + int_color = theme.fancycompleter.int + self.assertEqual(set(matches), { + ' ', + f'{N0}{int_color}foobar{ANSIColors.RESET}', + f'{N1}{int_color}foobazzz{ANSIColors.RESET}', + }) + self.assertEqual(compl.global_matches('foobaz'), ['foobazzz']) + self.assertEqual(compl.global_matches('nothing'), []) + + def test_large_color_sort_prefix_is_stripped(self): + compl = Completer({'a': 42}, use_colors=True) + match = compl._color_for_obj(1000, 'spam', 1) + self.assertEqual(stripcolor(match), 'spam') + + def test_complete_with_indexer(self): + compl = Completer({'lst': [None, 2, 3]}, use_colors=False) + self.assertEqual(compl.attr_matches('lst[0].'), ['lst[0].__']) + matches = compl.attr_matches('lst[0].__') + self.assertNotIn('__class__', matches) + self.assertIn('lst[0].__class__', matches) + match = compl.attr_matches('lst[0].__class') + self.assertEqual(len(match), 1) + self.assertTrue(match[0].startswith('lst[0].__class__')) + + def test_autocomplete(self): + class A: + aaa = None + abc_1 = None + abc_2 = None + abc_3 = None + bbb = None + compl = Completer({'A': A}, use_colors=False) + # + # In this case, we want to display all attributes which start with + # 'a'. Moreover, we also include a space to prevent readline to + # automatically insert the common prefix (which will the the ANSI escape + # sequence if we use colors). + matches = compl.attr_matches('A.a') + self.assertEqual( + sorted(matches), + [' ', 'A.aaa', 'A.abc_1', 'A.abc_2', 'A.abc_3'], + ) + # + # If there is an actual common prefix, we return just it, so that readline + # will insert it into place + matches = compl.attr_matches('A.ab') + self.assertEqual(matches, ['A.abc_']) + # + # Finally, at the next tab, we display again all the completions available + # for this common prefix. Again, we insert a spurious space to prevent the + # automatic completion of ANSI sequences. + matches = compl.attr_matches('A.abc_') + self.assertEqual( + sorted(matches), + [' ', 'A.abc_1', 'A.abc_2', 'A.abc_3'], + ) + + def test_complete_exception(self): + compl = Completer({}, use_colors=False) + self.assertEqual(compl.attr_matches('xxx.'), []) + + def test_complete_invalid_attr(self): + compl = Completer({'str': str}, use_colors=False) + self.assertEqual(compl.attr_matches('str.xx'), []) + + def test_complete_function_skipped(self): + compl = Completer({'str': str}, use_colors=False) + self.assertEqual(compl.attr_matches('str.split().'), []) + + def test_unicode_in___dir__(self): + class Foo(object): + def __dir__(self): + return ['hello', 'world'] + + compl = Completer({'a': Foo()}, use_colors=False) + matches = compl.attr_matches('a.') + self.assertEqual(matches, ['a.hello', 'a.world']) + self.assertIs(type(matches[0]), str) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 8854b19efce019..18e88ce4e7724a 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -33,6 +33,8 @@ ModuleCompleter, HARDCODED_SUBMODULES, ) +from _pyrepl.fancycompleter import Completer as FancyCompleter +import _pyrepl.readline as pyrepl_readline from _pyrepl.readline import ( ReadlineAlikeReader, ReadlineConfig, @@ -941,6 +943,92 @@ def test_func(self): self.assertEqual(mock_stderr.getvalue(), "") +class TestPyReplFancyCompleter(TestCase): + def prepare_reader(self, events, namespace, *, use_colors): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = FancyCompleter( + namespace, use_colors=use_colors + ).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_simple_completion_preserves_callable_postfix(self): + events = code_to_events("os.getpid\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace, use_colors=False) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getpid()") + + def test_attribute_menu_tracks_typed_stem(self): + class Obj: + apple = 1 + apricot = 2 + banana = 3 + + namespace = {"obj": Obj} + reader = self.prepare_reader( + code_to_events("obj.\t\ta"), + namespace, + use_colors=True, + ) + + with self.assertRaises(StopIteration): + while True: + reader.handle1() + + self.assertEqual("".join(reader.buffer), "obj.a") + self.assertTrue(reader.cmpltn_menu_visible) + menu = "\n".join(reader.cmpltn_menu) + self.assertIn("apple", menu) + self.assertIn("apricot", menu) + self.assertNotIn("banana", menu) + self.assertNotIn("mro", menu) + + +class TestPyReplReadlineSetup(TestCase): + def test_setup_ignores_basic_completer_env_when_env_is_disabled(self): + class FakeFancyCompleter: + def __init__(self, namespace): + self.namespace = namespace + + def complete(self, text, state): + return None + + class FakeBasicCompleter(FakeFancyCompleter): + pass + + wrapper = Mock() + wrapper.config = ReadlineConfig() + stdin = Mock() + stdout = Mock() + stdin.fileno.return_value = 0 + stdout.fileno.return_value = 1 + + with ( + patch.object(pyrepl_readline, "_wrapper", wrapper), + patch.object(pyrepl_readline, "raw_input", None), + patch.object(pyrepl_readline, "FancyCompleter", FakeFancyCompleter), + patch.object(pyrepl_readline, "RLCompleter", FakeBasicCompleter), + patch.object(pyrepl_readline.sys, "stdin", stdin), + patch.object(pyrepl_readline.sys, "stdout", stdout), + patch.object(pyrepl_readline.sys, "flags", Mock(ignore_environment=True)), + patch.object(pyrepl_readline.os, "isatty", return_value=True), + patch.object(pyrepl_readline.os, "getenv") as mock_getenv, + patch("builtins.input", lambda prompt="": prompt), + ): + mock_getenv.return_value = "1" + pyrepl_readline._setup({}) + + self.assertIsInstance( + wrapper.config.readline_completer.__self__, + FakeFancyCompleter, + ) + mock_getenv.assert_not_called() + + class TestPyReplModuleCompleter(TestCase): def setUp(self): # Make iter_modules() search only the standard library. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-52-31.gh-issue-144319.iZk4hs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-52-31.gh-issue-144319.iZk4hs.rst new file mode 100644 index 00000000000000..f3f07ab35dbb01 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-01-12-52-31.gh-issue-144319.iZk4hs.rst @@ -0,0 +1,3 @@ +Fix a bug that could cause applications with specific allocation patterns to +leak memory via Huge Pages if compiled with Huge Page support. Patch by +Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2025-09-19-13-54-54.gh-issue-130472.LODfdk.rst b/Misc/NEWS.d/next/Library/2025-09-19-13-54-54.gh-issue-130472.LODfdk.rst new file mode 100644 index 00000000000000..3d2a7f00d3e6a8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-19-13-54-54.gh-issue-130472.LODfdk.rst @@ -0,0 +1 @@ +Add fancycompleter and enable it by default when using pyrepl. This gives colored tab completion. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 983bdddbf026a8..e2d5b012955c3e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -14,6 +14,7 @@ #include // malloc() #include #include // fopen(), fgets(), sscanf() +#include // errno #ifdef WITH_MIMALLOC // Forward declarations of functions used in our mimalloc modifications static void _PyMem_mi_page_clear_qsbr(mi_page_t *page); @@ -572,6 +573,49 @@ _pymalloc_system_hugepage_size(void) } #endif +#if (defined(MS_WINDOWS) && defined(PYMALLOC_USE_HUGEPAGES)) || \ + (defined(PYMALLOC_USE_HUGEPAGES) && defined(ARENAS_USE_MMAP) && defined(MAP_HUGETLB)) +static size_t +_pymalloc_round_up_to_multiple(size_t size, size_t multiple) +{ + if (multiple == 0 || size == 0) { + return size; + } + + size_t remainder = size % multiple; + if (remainder == 0) { + return size; + } + + size_t padding = multiple - remainder; + if (size > SIZE_MAX - padding) { + return 0; + } + return size + padding; +} +#endif + +static size_t +_pymalloc_virtual_alloc_size(size_t size) +{ +#if defined(MS_WINDOWS) && defined(PYMALLOC_USE_HUGEPAGES) + if (_PyRuntime.allocators.use_hugepages) { + SIZE_T large_page_size = GetLargePageMinimum(); + if (large_page_size > 0) { + return _pymalloc_round_up_to_multiple(size, (size_t)large_page_size); + } + } +#elif defined(PYMALLOC_USE_HUGEPAGES) && defined(ARENAS_USE_MMAP) && defined(MAP_HUGETLB) + if (_PyRuntime.allocators.use_hugepages) { + size_t hp_size = _pymalloc_system_hugepage_size(); + if (hp_size > 0) { + return _pymalloc_round_up_to_multiple(size, hp_size); + } + } +#endif + return size; +} + void * _PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size) { @@ -648,7 +692,11 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, if (ptr == NULL) { return; } - munmap(ptr, size); + if (munmap(ptr, size) < 0) { + _Py_FatalErrorFormat(__func__, + "munmap(%p, %zu) failed with errno %d", + ptr, size, errno); + } #else free(ptr); #endif @@ -1128,13 +1176,19 @@ PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) void * _PyObject_VirtualAlloc(size_t size) { - return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size); + size_t alloc_size = _pymalloc_virtual_alloc_size(size); + if (alloc_size == 0 && size != 0) { + return NULL; + } + return _PyObject_Arena.alloc(_PyObject_Arena.ctx, alloc_size); } void _PyObject_VirtualFree(void *obj, size_t size) { - _PyObject_Arena.free(_PyObject_Arena.ctx, obj, size); + size_t alloc_size = _pymalloc_virtual_alloc_size(size); + assert(alloc_size != 0 || size == 0); + _PyObject_Arena.free(_PyObject_Arena.ctx, obj, alloc_size); }