[pull] main from python:main

Doc/using/cmdline.rst

@@ -1338,6 +1338,13 @@ conflict.
 
    .. versionadded:: 3.13
 
+.. envvar:: PYTHON_BASIC_COMPLETER
+
+   If this variable is set to any value, PyREPL will use :mod:`rlcompleter` to
+   implement tab completion, instead of the default one which uses colors.
+
+   .. versionadded:: 3.15
+
 .. envvar:: PYTHON_HISTORY
 
    This environment variable can be used to set the location of a

Include/internal/pycore_obmalloc.h

@@ -691,7 +691,11 @@ struct _obmalloc_state {
 
 
 /* Allocate memory directly from the O/S virtual memory system,
- * where supported. Otherwise fallback on malloc */
+ * where supported. Otherwise fallback on malloc.
+ *
+ * Large-page and huge-page backends may round the mapped size up
+ * internally, so pass the original requested size back to
+ * _PyObject_VirtualFree(). */
 void *_PyObject_VirtualAlloc(size_t size);
 void _PyObject_VirtualFree(void *, size_t size);
 

InternalDocs/interpreter.md

@@ -507,6 +507,38 @@ After the last `DEOPT_IF` has passed, a hit should be recorded with
 After an optimization has been deferred in the adaptive instruction,
 that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`.
 
+## Interpreter types
+There are three different types of interpreters to choose from based on compiler support:
+
+ * traditional switch-case interpreter
+
+   Supported by all compilers covered in PEP 7.
+
+ * computed-gotos interpreter
+
+   Enabled using configure option `--with-computed-gotos` and used by default on supported compilers.
+   It uses [Labels as Values](https://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html)
+   for more efficient dispatching.
+
+ * tail-calling interpreter
+
+   Enabled using configure option `--with-tail-call-interp` (or `--tail-call-interp` for build.bat on Windows).
+   It uses [tail calls](https://clang.llvm.org/docs/AttributeReference.html#musttail) and the
+   [preserve_none](https://clang.llvm.org/docs/AttributeReference.html#preserve-none)
+   calling convention between the small C functions that implement individual Python opcodes.
+
+   Not all compilers support these and if they do not all targets might be supported (for example,
+   MSVC currently only supports x64 and only in optimized builds).
+
+   In addition, compilers must do [escape analysis](https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-musttail)
+   of the lifetimes of automatic variables, function parameters, and temporaries to ensure proper tail-calls. They
+   emit a compile error in case of a violation or detection failure. The ability to detect this varies depending on the compiler and
+   also on the optimization level. Following techniques are particularly helpful to the MSVC compiler in this regard
+   * [Introducing additional scopes](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Python/bytecodes.c#L2526)
+   * [extracting problematic code paths into a separate function](https://github.com/python/cpython/pull/143068/files#diff-729a985b0cb8b431cb291f1edb561bbbfea22e3f8c262451cd83328a0936a342R3724)
+   * [returning a pointer instead of taking it as an output parameter](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Include/internal/pycore_ceval.h#L489-L492)
+
+   Using `restrict` is another (currently unused) remedy.
 
 Additional resources
 --------------------

Lib/_colorize.py

@@ -15,7 +15,6 @@
 
 class ANSIColors:
     RESET = "\x1b[0m"
-
     BLACK = "\x1b[30m"
     BLUE = "\x1b[34m"
     CYAN = "\x1b[36m"
@@ -200,6 +199,30 @@ class Difflib(ThemeSection):
     reset: str = ANSIColors.RESET
 
 
+@dataclass(frozen=True, kw_only=True)
+class FancyCompleter(ThemeSection):
+    # functions and methods
+    function: str = ANSIColors.BOLD_BLUE
+    builtin_function_or_method: str = ANSIColors.BOLD_BLUE
+    method: str = ANSIColors.BOLD_CYAN
+    method_wrapper: str = ANSIColors.BOLD_CYAN
+    wrapper_descriptor: str = ANSIColors.BOLD_CYAN
+    method_descriptor: str = ANSIColors.BOLD_CYAN
+
+    # numbers
+    int: str = ANSIColors.BOLD_YELLOW
+    float: str = ANSIColors.BOLD_YELLOW
+    complex: str = ANSIColors.BOLD_YELLOW
+    bool: str = ANSIColors.BOLD_YELLOW
+
+    # others
+    type: str = ANSIColors.BOLD_MAGENTA
+    module: str = ANSIColors.CYAN
+    NoneType: str = ANSIColors.GREY
+    bytes: str = ANSIColors.BOLD_GREEN
+    str: str = ANSIColors.BOLD_GREEN
+
+
 @dataclass(frozen=True, kw_only=True)
 class LiveProfiler(ThemeSection):
     """Theme section for the live profiling TUI (Tachyon profiler).
@@ -354,6 +377,7 @@ class Theme:
     """
     argparse: Argparse = field(default_factory=Argparse)
     difflib: Difflib = field(default_factory=Difflib)
+    fancycompleter: FancyCompleter = field(default_factory=FancyCompleter)
     live_profiler: LiveProfiler = field(default_factory=LiveProfiler)
     syntax: Syntax = field(default_factory=Syntax)
     traceback: Traceback = field(default_factory=Traceback)
@@ -364,6 +388,7 @@ def copy_with(
         *,
         argparse: Argparse | None = None,
         difflib: Difflib | None = None,
+        fancycompleter: FancyCompleter | None = None,
         live_profiler: LiveProfiler | None = None,
         syntax: Syntax | None = None,
         traceback: Traceback | None = None,
@@ -377,6 +402,7 @@ def copy_with(
         return type(self)(
             argparse=argparse or self.argparse,
             difflib=difflib or self.difflib,
+            fancycompleter=fancycompleter or self.fancycompleter,
             live_profiler=live_profiler or self.live_profiler,
             syntax=syntax or self.syntax,
             traceback=traceback or self.traceback,
@@ -394,6 +420,7 @@ def no_colors(cls) -> Self:
         return cls(
             argparse=Argparse.no_colors(),
             difflib=Difflib.no_colors(),
+            fancycompleter=FancyCompleter.no_colors(),
             live_profiler=LiveProfiler.no_colors(),
             syntax=Syntax.no_colors(),
             traceback=Traceback.no_colors(),

Lib/_pyrepl/completing_reader.py

@@ -178,12 +178,14 @@ def do(self) -> None:
         if not completions:
             r.error("no matches")
         elif len(completions) == 1:
-            if completions_unchangable and len(completions[0]) == len(stem):
+            completion = stripcolor(completions[0])
+            if completions_unchangable and len(completion) == len(stem):
                 r.msg = "[ sole completion ]"
                 r.dirty = True
-            r.insert(completions[0][len(stem):])
+            r.insert(completion[len(stem):])
         else:
-            p = prefix(completions, len(stem))
+            clean_completions = [stripcolor(word) for word in completions]
+            p = prefix(clean_completions, len(stem))
             if p:
                 r.insert(p)
             if last_is_completer:
@@ -195,7 +197,7 @@ def do(self) -> None:
                 r.dirty = True
             elif not r.cmpltn_menu_visible:
                 r.cmpltn_message_visible = True
-                if stem + p in completions:
+                if stem + p in clean_completions:
                     r.msg = "[ complete but not unique ]"
                     r.dirty = True
                 else:
@@ -215,7 +217,7 @@ def do(self) -> None:
                 r.cmpltn_reset()
             else:
                 completions = [w for w in r.cmpltn_menu_choices
-                               if w.startswith(stem)]
+                               if stripcolor(w).startswith(stem)]
                 if completions:
                     r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
                         r.console, completions, 0,

Lib/_pyrepl/fancycompleter.py

@@ -0,0 +1,210 @@
+#   Copyright 2010-2025 Antonio Cuni
+#                       Daniel Hahler
+#
+#                        All Rights Reserved
+"""Colorful tab completion for Python prompt"""
+from _colorize import ANSIColors, get_colors, get_theme
+import rlcompleter
+import keyword
+import types
+
+class Completer(rlcompleter.Completer):
+    """
+    When doing something like a.b.<tab>, keep the full a.b.attr completion
+    stem so readline-style completion can keep refining the menu as you type.
+
+    Optionally, display the various completions in different colors
+    depending on the type.
+    """
+    def __init__(
+        self,
+        namespace=None,
+        *,
+        use_colors='auto',
+        consider_getitems=True,
+    ):
+        from _pyrepl import readline
+        rlcompleter.Completer.__init__(self, namespace)
+        if use_colors == 'auto':
+            # use colors only if we can
+            use_colors = get_colors().RED != ""
+        self.use_colors = use_colors
+        self.consider_getitems = consider_getitems
+
+        if self.use_colors:
+            # In GNU readline, this prevents escaping of ANSI control
+            # characters in completion results. pyrepl's parse_and_bind()
+            # is a no-op, but pyrepl handles ANSI sequences natively
+            # via real_len()/stripcolor().
+            readline.parse_and_bind('set dont-escape-ctrl-chars on')
+            self.theme = get_theme()
+        else:
+            self.theme = None
+
+        if self.consider_getitems:
+            delims = readline.get_completer_delims()
+            delims = delims.replace('[', '')
+            delims = delims.replace(']', '')
+            readline.set_completer_delims(delims)
+
+    def complete(self, text, state):
+        # if you press <tab> at the beginning of a line, insert an actual
+        # \t. Else, trigger completion.
+        if text == "":
+            return ('\t', None)[state]
+        else:
+            return rlcompleter.Completer.complete(self, text, state)
+
+    def _callable_postfix(self, val, word):
+        # disable automatic insertion of '(' for global callables
+        return word
+
+    def _callable_attr_postfix(self, val, word):
+        return rlcompleter.Completer._callable_postfix(self, val, word)
+
+    def global_matches(self, text):
+        names = rlcompleter.Completer.global_matches(self, text)
+        prefix = commonprefix(names)
+        if prefix and prefix != text:
+            return [prefix]
+
+        names.sort()
+        values = []
+        for name in names:
+            clean_name = name.rstrip(': ')
+            if keyword.iskeyword(clean_name) or keyword.issoftkeyword(clean_name):
+                values.append(None)
+            else:
+                try:
+                    values.append(eval(name, self.namespace))
+                except Exception:
+                    values.append(None)
+        if self.use_colors and names:
+            return self.colorize_matches(names, values)
+        return names
+
+    def attr_matches(self, text):
+        try:
+            expr, attr, names, values = self._attr_matches(text)
+        except ValueError:
+            return []
+
+        if not names:
+            return []
+
+        if len(names) == 1:
+            # No coloring: when returning a single completion, readline
+            # inserts it directly into the prompt, so ANSI codes would
+            # appear as literal characters.
+            return [self._callable_attr_postfix(values[0], f'{expr}.{names[0]}')]
+
+        prefix = commonprefix(names)
+        if prefix and prefix != attr:
+            return [f'{expr}.{prefix}']  # autocomplete prefix
+
+        names = [f'{expr}.{name}' for name in names]
+        if self.use_colors:
+            return self.colorize_matches(names, values)
+
+        if prefix:
+            names.append(' ')
+        return names
+
+    def _attr_matches(self, text):
+        expr, attr = text.rsplit('.', 1)
+        if '(' in expr or ')' in expr:  # don't call functions
+            return expr, attr, [], []
+        try:
+            thisobject = eval(expr, self.namespace)
+        except Exception:
+            return expr, attr, [], []
+
+        # get the content of the object, except __builtins__
+        words = set(dir(thisobject)) - {'__builtins__'}
+
+        if hasattr(thisobject, '__class__'):
+            words.add('__class__')
+            words.update(rlcompleter.get_class_members(thisobject.__class__))
+        names = []
+        values = []
+        n = len(attr)
+        if attr == '':
+            noprefix = '_'
+        elif attr == '_':
+            noprefix = '__'
+        else:
+            noprefix = None
+
+        # sort the words now to make sure to return completions in
+        # alphabetical order. It's easier to do it now, else we would need to
+        # sort 'names' later but make sure that 'values' in kept in sync,
+        # which is annoying.
+        words = sorted(words)
+        while True:
+            for word in words:
+                if (
+                    word[:n] == attr
+                    and not (noprefix and word[:n+1] == noprefix)
+                ):
+                    # Mirror rlcompleter's safeguards so completion does not
+                    # call properties or reify lazy module attributes.
+                    if isinstance(getattr(type(thisobject), word, None), property):
+                        value = None
+                    elif (
+                        isinstance(thisobject, types.ModuleType)
+                        and isinstance(
+                            thisobject.__dict__.get(word),
+                            types.LazyImportType,
+                        )
+                    ):
+                        value = thisobject.__dict__.get(word)
+                    else:
+                        value = getattr(thisobject, word, None)
+
+                    names.append(word)
+                    values.append(value)
+            if names or not noprefix:
+                break
+            if noprefix == '_':
+                noprefix = '__'
+            else:
+                noprefix = None
+
+        return expr, attr, names, values
+
+    def colorize_matches(self, names, values):
+        matches = [self._color_for_obj(i, name, obj)
+                   for i, (name, obj)
+                   in enumerate(zip(names, values))]
+        # We add a space at the end to prevent the automatic completion of the
+        # common prefix, which is the ANSI escape sequence.
+        matches.append(' ')
+        return matches
+
+    def _color_for_obj(self, i, name, value):
+        t = type(value)
+        color = self._color_by_type(t)
+        # Encode the match index into a fake escape sequence that
+        # stripcolor() can still remove once i reaches four digits.
+        N = f"\x1b[{i // 100:03d};{i % 100:02d}m"
+        return f"{N}{color}{name}{ANSIColors.RESET}"
+
+    def _color_by_type(self, t):
+        typename = t.__name__
+        # this is needed e.g. to turn method-wrapper into method_wrapper,
+        # because if we want _colorize.FancyCompleter to be "dataclassable"
+        # our keys need to be valid identifiers.
+        typename = typename.replace('-', '_').replace('.', '_')
+        return getattr(self.theme.fancycompleter, typename, ANSIColors.RESET)
+
+
+def commonprefix(names):
+    """Return the common prefix of all 'names'"""
+    if not names:
+        return ''
+    s1 = min(names)
+    s2 = max(names)
+    for i, c in enumerate(s1):
+        if c != s2[i]:
+            return s1[:i]
+    return s1