mirror of https://github.com/python/peps
686 lines
45 KiB
HTML
686 lines
45 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 638 – Syntactic Macros | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0638/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 638 – Syntactic Macros | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0638/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 638</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 638 – Syntactic Macros</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Mark Shannon <mark at hotpy.org></dd>
|
||
<dt class="field-even">Discussions-To<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/U4C4XHNRC4SHS3TPZWCTY4SN4QU3TT6V/">Python-Dev thread</a></dd>
|
||
<dt class="field-odd">Status<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Proposal under active discussion and revision">Draft</abbr></dd>
|
||
<dt class="field-even">Type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-odd">Created<span class="colon">:</span></dt>
|
||
<dd class="field-odd">24-Sep-2020</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">26-Sep-2020</dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a><ul>
|
||
<li><a class="reference internal" href="#improving-the-expressiveness-of-libraries-for-specific-domains">Improving the expressiveness of libraries for specific domains</a></li>
|
||
<li><a class="reference internal" href="#trialing-new-language-features">Trialing new language features</a></li>
|
||
<li><a class="reference internal" href="#long-term-stability-for-the-bytecode-interpreter">Long term stability for the bytecode interpreter</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#syntax">Syntax</a><ul>
|
||
<li><a class="reference internal" href="#lexical-analysis">Lexical analysis</a></li>
|
||
<li><a class="reference internal" href="#statement-form">Statement form</a></li>
|
||
<li><a class="reference internal" href="#expression-form">Expression form</a></li>
|
||
<li><a class="reference internal" href="#resolving-ambiguity">Resolving ambiguity</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#semantics">Semantics</a><ul>
|
||
<li><a class="reference internal" href="#compilation">Compilation</a></li>
|
||
<li><a class="reference internal" href="#defining-macro-processors">Defining macro processors</a></li>
|
||
<li><a class="reference internal" href="#ast-extensions">AST extensions</a></li>
|
||
<li><a class="reference internal" href="#hygiene-and-debugging">Hygiene and debugging</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#examples">Examples</a><ul>
|
||
<li><a class="reference internal" href="#compile-time-checked-data-structures">Compile-time-checked data structures</a></li>
|
||
<li><a class="reference internal" href="#domain-specific-extensions">Domain-specific extensions</a></li>
|
||
<li><a class="reference internal" href="#compilers">Compilers</a></li>
|
||
<li><a class="reference internal" href="#matching-symbolic-expressions">Matching symbolic expressions</a></li>
|
||
<li><a class="reference internal" href="#zero-cost-markers-and-annotations">Zero-cost markers and annotations</a></li>
|
||
<li><a class="reference internal" href="#prototyping-language-extensions">Prototyping language extensions</a><ul>
|
||
<li><a class="reference internal" href="#f-strings">f-strings:</a></li>
|
||
<li><a class="reference internal" href="#try-finally-statement">Try finally statement:</a></li>
|
||
<li><a class="reference internal" href="#with-statement">With statement:</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#macro-definition-macros">Macro definition macros</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#performance-implications">Performance Implications</a></li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a><ul>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP adds support for syntactic macros to Python.
|
||
A macro is a compile-time function that transforms
|
||
a part of the program to allow functionality that cannot be
|
||
expressed cleanly in normal library code.</p>
|
||
<p>The term “syntactic” means that this sort of macro operates on the program’s
|
||
syntax tree. This reduces the chance of mistranslation that can happen
|
||
with text-based substitution macros, and allows the implementation
|
||
of <a class="reference external" href="https://en.wikipedia.org/wiki/Hygienic_macro">hygienic macros</a>.</p>
|
||
<p>Syntactic macros allow libraries to modify the abstract syntax tree during compilation,
|
||
providing the ability to extend the language for specific domains without
|
||
adding to complexity to the language as a whole.</p>
|
||
</section>
|
||
<section id="motivation">
|
||
<h2><a class="toc-backref" href="#motivation" role="doc-backlink">Motivation</a></h2>
|
||
<p>New language features can be controversial, disruptive and sometimes divisive.
|
||
Python is now sufficiently powerful and complex, that many proposed additions
|
||
are a net loss for the language due to the additional complexity.</p>
|
||
<p>Although a language change may make certain patterns easy to express,
|
||
it will have a cost. Each new feature makes the language larger,
|
||
harder to learn and harder to understand.
|
||
Python was once described as <a class="reference external" href="https://www.linuxjournal.com/article/4731">Python Fits Your Brain</a>,
|
||
but that becomes less and less true as more and more features are added.</p>
|
||
<p>Because of the high cost of adding a new feature,
|
||
it is very difficult or impossible to add a feature that would benefit only
|
||
some users, regardless of how many users, or how beneficial that feature would
|
||
be to them.</p>
|
||
<p>The use of Python in data science and machine learning has grown very rapidly
|
||
over the last few years.
|
||
However, most of the core developers of Python do not have a background in
|
||
data science or machine learning.
|
||
This makes it extremely difficult for the core developers to determine whether a
|
||
language extension for machine learning is worthwhile.</p>
|
||
<p>By allowing language extensions to be modular and distributable, like libraries,
|
||
domain-specific extensions can be implemented without negatively impacting
|
||
users outside of that domain.
|
||
A web developer is likely to want a very different set of extensions from
|
||
a data scientist.
|
||
We need to let the community develop their own extensions.</p>
|
||
<p>Without some form of user-defined language extensions,
|
||
there will be a constant battle between those wanting to keep the
|
||
language compact and fitting their brains, and those wanting a new feature
|
||
that suits their domain or programming style.</p>
|
||
<section id="improving-the-expressiveness-of-libraries-for-specific-domains">
|
||
<h3><a class="toc-backref" href="#improving-the-expressiveness-of-libraries-for-specific-domains" role="doc-backlink">Improving the expressiveness of libraries for specific domains</a></h3>
|
||
<p>Many domains see repeated patterns that are difficult or impossible
|
||
to express as a library.
|
||
Macros can express those patterns in a more concise and less error-prone way.</p>
|
||
</section>
|
||
<section id="trialing-new-language-features">
|
||
<h3><a class="toc-backref" href="#trialing-new-language-features" role="doc-backlink">Trialing new language features</a></h3>
|
||
<p>It is possible to demonstrate potential language extensions using macros.
|
||
For example, macros would have enabled the <code class="docutils literal notranslate"><span class="pre">with</span></code> statement and
|
||
<code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code> expression to have been trialed.
|
||
Doing so might well have lead to a higher quality implementation
|
||
at first release, by allowing more testing
|
||
before those features were included in the language.</p>
|
||
<p>It is nearly impossible to make sure that a new feature is completely reliable
|
||
before it is released; bugs relating to the <code class="docutils literal notranslate"><span class="pre">with</span></code> and <code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code>
|
||
features were still being fixed many years after they were released.</p>
|
||
</section>
|
||
<section id="long-term-stability-for-the-bytecode-interpreter">
|
||
<h3><a class="toc-backref" href="#long-term-stability-for-the-bytecode-interpreter" role="doc-backlink">Long term stability for the bytecode interpreter</a></h3>
|
||
<p>Historically, new language features have been implemented by naive compilation
|
||
of the AST into new, complex bytecode instructions.
|
||
Those bytecodes have often had their own internal flow-control, performing
|
||
operations that could, and should, have been done in the compiler.</p>
|
||
<p>For example,
|
||
until recently flow control within the <code class="docutils literal notranslate"><span class="pre">try</span></code>-<code class="docutils literal notranslate"><span class="pre">finally</span></code> and <code class="docutils literal notranslate"><span class="pre">with</span></code>
|
||
statements was managed by complicated bytecodes with context-dependent semantics.
|
||
The control flow within those statements is now implemented in the compiler, making
|
||
the interpreter simpler and faster.</p>
|
||
<p>By implementing new features as AST transformations, the existing compiler can
|
||
generate the bytecode for a feature without having to modify the interpreter.</p>
|
||
<p>A stable interpreter is necessary if we are to improve the performance and
|
||
portability of the CPython VM.</p>
|
||
</section>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Python is both expressive and easy to learn;
|
||
it is widely recognized as the easiest-to-learn, widely used programming language.
|
||
However, it is not the most flexible. That title belongs to lisp.</p>
|
||
<p>Because lisp is homoiconic, meaning that lisp programs are lisp data structures,
|
||
lisp programs can be manipulated by lisp programs.
|
||
Thus much of the language can be defined in itself.</p>
|
||
<p>We would like that ability in Python,
|
||
without the many parentheses that characterize lisp.
|
||
Fortunately, homoiconicity is not needed for a language to be able to
|
||
manipulate itself, all that is needed is the ability to manipulate programs
|
||
after parsing, but before translation to an executable form.</p>
|
||
<p>Python already has the components needed.
|
||
The syntax tree of Python is available through the <code class="docutils literal notranslate"><span class="pre">ast</span></code> module.
|
||
All that is needed is a marker to tell the compiler that a macro is present,
|
||
and the ability for the compiler to callback into user code to manipulate the AST.</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<section id="syntax">
|
||
<h3><a class="toc-backref" href="#syntax" role="doc-backlink">Syntax</a></h3>
|
||
<section id="lexical-analysis">
|
||
<h4><a class="toc-backref" href="#lexical-analysis" role="doc-backlink">Lexical analysis</a></h4>
|
||
<p>Any sequence of identifier characters followed by an exclamation point
|
||
(exclamation mark, UK English) will be tokenized as a <code class="docutils literal notranslate"><span class="pre">MACRO_NAME</span></code>.</p>
|
||
</section>
|
||
<section id="statement-form">
|
||
<h4><a class="toc-backref" href="#statement-form" role="doc-backlink">Statement form</a></h4>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">macro_stmt</span> <span class="o">=</span> <span class="n">MACRO_NAME</span> <span class="n">testlist</span> <span class="p">[</span> <span class="s2">"import"</span> <span class="n">NAME</span> <span class="p">]</span> <span class="p">[</span> <span class="s2">"as"</span> <span class="n">NAME</span> <span class="p">]</span> <span class="p">[</span> <span class="s2">":"</span> <span class="n">NEWLINE</span> <span class="n">suite</span> <span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="expression-form">
|
||
<h4><a class="toc-backref" href="#expression-form" role="doc-backlink">Expression form</a></h4>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">macro_expr</span> <span class="o">=</span> <span class="n">MACRO_NAME</span> <span class="s2">"("</span> <span class="n">testlist</span> <span class="s2">")"</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="resolving-ambiguity">
|
||
<h4><a class="toc-backref" href="#resolving-ambiguity" role="doc-backlink">Resolving ambiguity</a></h4>
|
||
<p>The statement form of a macro takes precedence, so that the code
|
||
<code class="docutils literal notranslate"><span class="pre">macro_name!(x)</span></code> will be parsed as a macro statement,
|
||
not as an expression statement containing a macro expression.</p>
|
||
</section>
|
||
</section>
|
||
<section id="semantics">
|
||
<h3><a class="toc-backref" href="#semantics" role="doc-backlink">Semantics</a></h3>
|
||
<section id="compilation">
|
||
<h4><a class="toc-backref" href="#compilation" role="doc-backlink">Compilation</a></h4>
|
||
<p>Upon encountering a <code class="docutils literal notranslate"><span class="pre">macro</span></code> during translation to bytecode,
|
||
the code generator will look up the macro processor registered for the macro,
|
||
and pass the AST rooted at the macro to the processor function.
|
||
The returned AST will then be substituted for the original tree.</p>
|
||
<p>For macros with multiple names,
|
||
several trees will be passed to the macro processor,
|
||
but only one will be returned and substituted,
|
||
shorting the enclosing block of statements.</p>
|
||
<p>This process can be repeated,
|
||
to enable macros to return AST nodes including other macros.</p>
|
||
<p>The compiler will not look up a macro processor until that macro is reached,
|
||
so that inner macros do not need to have processors registered.
|
||
For example, in a <code class="docutils literal notranslate"><span class="pre">switch</span></code> macro, the <code class="docutils literal notranslate"><span class="pre">case</span></code> and <code class="docutils literal notranslate"><span class="pre">default</span></code> macros wouldn’t
|
||
need processors registered as they would be eliminated by the <code class="docutils literal notranslate"><span class="pre">switch</span></code> processor.</p>
|
||
<p>To enable definition of macros to be imported,
|
||
the macros <code class="docutils literal notranslate"><span class="pre">import!</span></code> and <code class="docutils literal notranslate"><span class="pre">from!</span></code> are predefined.
|
||
They support the following syntax:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="s2">"import!"</span> <span class="n">dotted_name</span> <span class="s2">"as"</span> <span class="n">name</span>
|
||
|
||
<span class="s2">"from!"</span> <span class="n">dotted_name</span> <span class="s2">"import"</span> <span class="n">name</span> <span class="p">[</span> <span class="s2">"as"</span> <span class="n">name</span> <span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">import!</span></code> macro performs a compile-time import of <code class="docutils literal notranslate"><span class="pre">dotted_name</span></code>
|
||
to find the macro processor, then registers it under <code class="docutils literal notranslate"><span class="pre">name</span></code>
|
||
for the scope currently being compiled.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">from!</span></code> macro performs a compile-time import of <code class="docutils literal notranslate"><span class="pre">dotted_name.name</span></code>
|
||
to find the macro processor, then registers it under <code class="docutils literal notranslate"><span class="pre">name</span></code>
|
||
(using the <code class="docutils literal notranslate"><span class="pre">name</span></code> following “as”, if present)
|
||
for the scope currently being compiled.</p>
|
||
<p>Note that, since <code class="docutils literal notranslate"><span class="pre">import!</span></code> and <code class="docutils literal notranslate"><span class="pre">from!</span></code> only define the macro for the
|
||
scope in which the import is present, all uses of a macro must be preceded by
|
||
an explicit <code class="docutils literal notranslate"><span class="pre">import!</span></code> or <code class="docutils literal notranslate"><span class="pre">from!</span></code> to improve clarity.</p>
|
||
<p>For example, to import the macro “compile” from “my.compiler”:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>from! my.compiler import compile
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="defining-macro-processors">
|
||
<h4><a class="toc-backref" href="#defining-macro-processors" role="doc-backlink">Defining macro processors</a></h4>
|
||
<p>A macro processor is defined by a four-tuple, consisting of
|
||
<code class="docutils literal notranslate"><span class="pre">(func,</span> <span class="pre">kind,</span> <span class="pre">version,</span> <span class="pre">additional_names)</span></code>:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">func</span></code> must be a callable that takes <code class="docutils literal notranslate"><span class="pre">len(additional_names)+1</span></code> arguments, all of which are abstract syntax trees, and returns a single abstract syntax tree.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">kind</span></code> must be one of the following:<ul>
|
||
<li><code class="docutils literal notranslate"><span class="pre">macros.STMT_MACRO</span></code>: A statement macro where the body of the macro is indented. This is the only form allowed to have additional names.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">macros.SIBLING_MACRO</span></code>: A statement macro where the body of the macro is the next statement in the same block. The following statement is moved into the macro as its body.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">macros.EXPR_MACRO</span></code>: An expression macro.</li>
|
||
</ul>
|
||
</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">version</span></code> is used to track versions of macros, so that generated bytecodes can be correctly cached. It must be an integer.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">additional_names</span></code> are the names of the additional parts of the macro, and must be a tuple of strings.</li>
|
||
</ul>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span># (func, _ast.STMT_MACRO, VERSION, ())
|
||
stmt_macro!:
|
||
multi_statement_body
|
||
|
||
# (func, _ast.SIBLING_MACRO, VERSION, ())
|
||
sibling_macro!
|
||
single_statement_body
|
||
|
||
# (func, _ast.EXPR_MACRO, VERSION, ())
|
||
x = expr_macro!(...)
|
||
|
||
# (func, _ast.STMT_MACRO, VERSION, ("subsequent_macro_part",))
|
||
multi_part_macro!:
|
||
multi_statement_body
|
||
subsequent_macro_part!:
|
||
multi_statement_body
|
||
</pre></div>
|
||
</div>
|
||
<p>The compiler will check that the syntax used matches the declared kind.</p>
|
||
<p>For convenience, the decorator <code class="docutils literal notranslate"><span class="pre">macro_processor</span></code> is provided in the <code class="docutils literal notranslate"><span class="pre">macros</span></code> module to mark a function as a macro processor:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">macro_processor</span><span class="p">(</span><span class="n">kind</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="o">*</span><span class="n">additional_names</span><span class="p">):</span>
|
||
<span class="k">def</span> <span class="nf">deco</span><span class="p">(</span><span class="n">func</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">func</span><span class="p">,</span> <span class="n">kind</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">additional_names</span>
|
||
<span class="k">return</span> <span class="n">deco</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Which can be used to help declare macro processors, for example:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nd">@macros</span><span class="o">.</span><span class="n">macro_processor</span><span class="p">(</span><span class="n">macros</span><span class="o">.</span><span class="n">STMT_MACRO</span><span class="p">,</span> <span class="mi">1_08</span><span class="p">)</span>
|
||
<span class="k">def</span> <span class="nf">switch</span><span class="p">(</span><span class="n">astnode</span><span class="p">):</span>
|
||
<span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="ast-extensions">
|
||
<h4><a class="toc-backref" href="#ast-extensions" role="doc-backlink">AST extensions</a></h4>
|
||
<p>Two new AST nodes will be needed to express macros, <code class="docutils literal notranslate"><span class="pre">macro_stmt</span></code> and <code class="docutils literal notranslate"><span class="pre">macro_expr</span></code>.</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">macro_stmt</span><span class="p">(</span><span class="n">_ast</span><span class="o">.</span><span class="n">stmt</span><span class="p">):</span>
|
||
<span class="n">_fields</span> <span class="o">=</span> <span class="s2">"name"</span><span class="p">,</span> <span class="s2">"args"</span><span class="p">,</span> <span class="s2">"importname"</span><span class="p">,</span> <span class="s2">"asname"</span><span class="p">,</span> <span class="s2">"body"</span>
|
||
|
||
<span class="k">class</span> <span class="nc">macro_expr</span><span class="p">(</span><span class="n">_ast</span><span class="o">.</span><span class="n">expr</span><span class="p">):</span>
|
||
<span class="n">_fields</span> <span class="o">=</span> <span class="s2">"name"</span><span class="p">,</span> <span class="s2">"args"</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>In addition, macro processors will need a means to express control flow or side-effecting code, that produces a value.
|
||
A new AST node called <code class="docutils literal notranslate"><span class="pre">stmt_expr</span></code> will be added, combining a statement and an expression.
|
||
This new ast node will be a subtype of <code class="docutils literal notranslate"><span class="pre">expr</span></code>, but include a statement to allow side effects.
|
||
It will be compiled to bytecode by compiling the statement, then compiling the value.</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">stmt_expr</span><span class="p">(</span><span class="n">_ast</span><span class="o">.</span><span class="n">expr</span><span class="p">):</span>
|
||
<span class="n">_fields</span> <span class="o">=</span> <span class="s2">"stmt"</span><span class="p">,</span> <span class="s2">"value"</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="hygiene-and-debugging">
|
||
<h4><a class="toc-backref" href="#hygiene-and-debugging" role="doc-backlink">Hygiene and debugging</a></h4>
|
||
<p>Macro processors will often need to create new variables.
|
||
Those variables need to named in such as way as to avoid contaminating the original code and other macros.
|
||
No rules for naming will be enforced, but to ensure hygiene and help debugging, the following naming scheme is recommended:</p>
|
||
<ul class="simple">
|
||
<li>All generated variable names should start with a <code class="docutils literal notranslate"><span class="pre">$</span></code></li>
|
||
<li>Purely artificial variable names should start <code class="docutils literal notranslate"><span class="pre">$$mname</span></code> where <code class="docutils literal notranslate"><span class="pre">mname</span></code> is the name of the macro.</li>
|
||
<li>Variables derived from real variables should start <code class="docutils literal notranslate"><span class="pre">$vname</span></code> where <code class="docutils literal notranslate"><span class="pre">vname</span></code> is the name of the variable.</li>
|
||
<li>All variable names should include the line number and the column offset, separated by an underscore.</li>
|
||
</ul>
|
||
<p>Examples:</p>
|
||
<ul class="simple">
|
||
<li>Purely generated name: <code class="docutils literal notranslate"><span class="pre">$$macro_17_0</span></code></li>
|
||
<li>Name derived from a variable for an expression macro: <code class="docutils literal notranslate"><span class="pre">$var_12_5</span></code></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="examples">
|
||
<h3><a class="toc-backref" href="#examples" role="doc-backlink">Examples</a></h3>
|
||
<section id="compile-time-checked-data-structures">
|
||
<h4><a class="toc-backref" href="#compile-time-checked-data-structures" role="doc-backlink">Compile-time-checked data structures</a></h4>
|
||
<p>It is common to encode tables of data in Python as large dictionaries.
|
||
However, these can be hard to maintain and error prone.
|
||
Macros allow such data to be written in a more readable format.
|
||
Then, at compile time, the data can be verified and converted to an efficient format.</p>
|
||
<p>For example, suppose we have a two dictionary literals mapping codes to names,
|
||
and vice versa.
|
||
This is error prone, as the dictionaries may have duplicate keys,
|
||
or one table may not be the inverse of the other.
|
||
A macro could generate the two mappings from a single table and,
|
||
at the same time, verify that no duplicates are present.</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">color_to_code</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s2">"red"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"blue"</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
|
||
<span class="s2">"green"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">code_to_color</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="mi">1</span><span class="p">:</span> <span class="s2">"red"</span><span class="p">,</span>
|
||
<span class="mi">2</span><span class="p">:</span> <span class="s2">"blue"</span><span class="p">,</span>
|
||
<span class="mi">3</span><span class="p">:</span> <span class="s2">"yellow"</span><span class="p">,</span> <span class="c1"># error</span>
|
||
<span class="p">}</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>would become:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>bijection! color_to_code, code_to_color:
|
||
"red" = 1
|
||
"blue" = 2
|
||
"green" = 3
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="domain-specific-extensions">
|
||
<h4><a class="toc-backref" href="#domain-specific-extensions" role="doc-backlink">Domain-specific extensions</a></h4>
|
||
<p>Where I see macros having real value is in specific domains, not in general-purpose language features.</p>
|
||
<p>For example, parsers.
|
||
Here’s part of a parser definition for Python, using macros:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>choice! single_input:
|
||
NEWLINE
|
||
simple_stmt
|
||
sequence!:
|
||
compound_stmt
|
||
NEWLINE
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="compilers">
|
||
<h4><a class="toc-backref" href="#compilers" role="doc-backlink">Compilers</a></h4>
|
||
<p>Runtime compilers, such as <code class="docutils literal notranslate"><span class="pre">numba</span></code> have to reconstitute the Python source, or attempt to analyze the bytecode.
|
||
It would be simpler and more reliable for them to get the AST directly:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>from! my.jit.library import jit
|
||
|
||
jit!
|
||
def func():
|
||
...
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="matching-symbolic-expressions">
|
||
<h4><a class="toc-backref" href="#matching-symbolic-expressions" role="doc-backlink">Matching symbolic expressions</a></h4>
|
||
<p>When matching something representing syntax, such a Python <code class="docutils literal notranslate"><span class="pre">ast</span></code> node, or a <code class="docutils literal notranslate"><span class="pre">sympy</span></code> expression,
|
||
it is convenient to match against the actual syntax, not the data structure representing it.
|
||
For example, a calculator could be implemented using a domain-specific macro for matching syntax:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>from! ast_matcher import match
|
||
|
||
def calculate(node):
|
||
if isinstance(node, Num):
|
||
return node.n
|
||
match! node:
|
||
case! a + b:
|
||
return calculate(a) + calculate(b)
|
||
case! a - b:
|
||
return calculate(a) - calculate(b)
|
||
case! a * b:
|
||
return calculate(a) * calculate(b)
|
||
case! a / b:
|
||
return calculate(a) / calculate(b)
|
||
</pre></div>
|
||
</div>
|
||
<p>Which could be converted to:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>def calculate(node):
|
||
if isinstance(node, Num):
|
||
return node.n
|
||
$$match_4_0 = node
|
||
if isinstance($$match_4_0, _ast.Add):
|
||
a, b = $$match_4_0.left, $$match_4_0.right
|
||
return calculate(a) + calculate(b)
|
||
elif isinstance($$match_4_0, _ast.Sub):
|
||
a, b = $$match_4_0.left, $$match_4_0.right
|
||
return calculate(a) - calculate(b)
|
||
elif isinstance($$match_4_0, _ast.Mul):
|
||
a, b = $$match_4_0.left, $$match_4_0.right
|
||
return calculate(a) * calculate(b)
|
||
elif isinstance($$match_4_0, _ast.Div):
|
||
a, b = $$match_4_0.left, $$match_4_0.right
|
||
return calculate(a) / calculate(b)
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="zero-cost-markers-and-annotations">
|
||
<h4><a class="toc-backref" href="#zero-cost-markers-and-annotations" role="doc-backlink">Zero-cost markers and annotations</a></h4>
|
||
<p>Annotations, either decorators or <a class="pep reference internal" href="../pep-3107/" title="PEP 3107 – Function Annotations">PEP 3107</a> function annotations, have a runtime cost
|
||
even if they serve only as markers for checkers or as documentation.</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nd">@do_nothing_marker</span>
|
||
<span class="k">def</span> <span class="nf">foo</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
||
<span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>can be replaced with the zero-cost macro:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>do_nothing_marker!:
|
||
def foo(...):
|
||
...
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="prototyping-language-extensions">
|
||
<h4><a class="toc-backref" href="#prototyping-language-extensions" role="doc-backlink">Prototyping language extensions</a></h4>
|
||
<p>Although macros would be most valuable for domain-specific extensions, it is possible to
|
||
demonstrate possible language extensions using macros.</p>
|
||
<section id="f-strings">
|
||
<h5><a class="toc-backref" href="#f-strings" role="doc-backlink">f-strings:</a></h5>
|
||
<p>The f-string <code class="docutils literal notranslate"><span class="pre">f"..."</span></code> could be implemented as macro as <code class="docutils literal notranslate"><span class="pre">f!("...")</span></code>.
|
||
Not quite as nice to read, but would still be useful for experimenting with.</p>
|
||
</section>
|
||
<section id="try-finally-statement">
|
||
<h5><a class="toc-backref" href="#try-finally-statement" role="doc-backlink">Try finally statement:</a></h5>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>try_!:
|
||
body
|
||
finally!:
|
||
closing
|
||
</pre></div>
|
||
</div>
|
||
<p>Would be translated roughly as:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">try</span><span class="p">:</span>
|
||
<span class="n">body</span>
|
||
<span class="k">except</span><span class="p">:</span>
|
||
<span class="n">closing</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">closing</span>
|
||
</pre></div>
|
||
</div>
|
||
<dl class="simple">
|
||
<dt>Note:</dt><dd>Care must be taken to handle returns, breaks and continues correctly.
|
||
The above code is merely illustrative.</dd>
|
||
</dl>
|
||
</section>
|
||
<section id="with-statement">
|
||
<h5><a class="toc-backref" href="#with-statement" role="doc-backlink">With statement:</a></h5>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>with! open(filename) as fd:
|
||
return fd.read()
|
||
</pre></div>
|
||
</div>
|
||
<p>The above would require handling <code class="docutils literal notranslate"><span class="pre">open</span></code> specially.
|
||
An alternative that would be more explicit, would be:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>with! open!(filename) as fd:
|
||
return fd.read()
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="macro-definition-macros">
|
||
<h4><a class="toc-backref" href="#macro-definition-macros" role="doc-backlink">Macro definition macros</a></h4>
|
||
<p>Languages that have syntactic macros usually provide a macro for defining macros.
|
||
This PEP intentionally does not do that, as it is not yet clear what a good design
|
||
would be, and we want to allow the community to define their own macros.</p>
|
||
<p>One possible form could be:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>macro_def! name:
|
||
input:
|
||
... # input pattern, defining meta-variables
|
||
output:
|
||
... # output pattern, using meta-variables
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards Compatibility</a></h2>
|
||
<p>This PEP is fully backwards compatible.</p>
|
||
</section>
|
||
<section id="performance-implications">
|
||
<h2><a class="toc-backref" href="#performance-implications" role="doc-backlink">Performance Implications</a></h2>
|
||
<p>For code that doesn’t use macros, there will be no effect on performance.</p>
|
||
<p>For code that does use macros and has already been compiled to bytecode,
|
||
there will be some slight overhead to check that the version
|
||
of macros used to compile the code match the imported macro processors.</p>
|
||
<p>For code that has not been compiled, or compiled with different versions
|
||
of the macro processors, then there would be the usual overhead of bytecode
|
||
compilation, plus any additional overhead of macro processing.</p>
|
||
<p>It is worth noting that the speed of source to bytecode compilation
|
||
is largely irrelevant for Python performance.</p>
|
||
</section>
|
||
<section id="implementation">
|
||
<h2><a class="toc-backref" href="#implementation" role="doc-backlink">Implementation</a></h2>
|
||
<p>In order to allow transformation of the AST at compile time by Python code,
|
||
all AST nodes in the compiler will have to be Python objects.</p>
|
||
<p>To do that efficiently, will mean making all the nodes in the <code class="docutils literal notranslate"><span class="pre">_ast</span></code> module
|
||
immutable, so as not degrade performance by much.
|
||
They will need to be immutable to guarantee that the AST remains a <em>tree</em>
|
||
to avoid having to support cyclic GC.
|
||
Making them immutable means they will not have a
|
||
<code class="docutils literal notranslate"><span class="pre">__dict__</span></code> attribute, making them compact.</p>
|
||
<p>AST nodes in the <code class="docutils literal notranslate"><span class="pre">ast</span></code> module will remain mutable.</p>
|
||
<p>Currently, all AST nodes are allocated using an arena allocator.
|
||
Changing to use the standard allocator might slow compilation down a little,
|
||
but has advantages in terms of maintenance, as much code can be deleted.</p>
|
||
<section id="reference-implementation">
|
||
<h3><a class="toc-backref" href="#reference-implementation" role="doc-backlink">Reference Implementation</a></h3>
|
||
<p>None as yet.</p>
|
||
</section>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document is placed in the public domain or under the
|
||
CC0-1.0-Universal license, whichever is more permissive.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0638.rst">https://github.com/python/peps/blob/main/peps/pep-0638.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0638.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a><ul>
|
||
<li><a class="reference internal" href="#improving-the-expressiveness-of-libraries-for-specific-domains">Improving the expressiveness of libraries for specific domains</a></li>
|
||
<li><a class="reference internal" href="#trialing-new-language-features">Trialing new language features</a></li>
|
||
<li><a class="reference internal" href="#long-term-stability-for-the-bytecode-interpreter">Long term stability for the bytecode interpreter</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#syntax">Syntax</a><ul>
|
||
<li><a class="reference internal" href="#lexical-analysis">Lexical analysis</a></li>
|
||
<li><a class="reference internal" href="#statement-form">Statement form</a></li>
|
||
<li><a class="reference internal" href="#expression-form">Expression form</a></li>
|
||
<li><a class="reference internal" href="#resolving-ambiguity">Resolving ambiguity</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#semantics">Semantics</a><ul>
|
||
<li><a class="reference internal" href="#compilation">Compilation</a></li>
|
||
<li><a class="reference internal" href="#defining-macro-processors">Defining macro processors</a></li>
|
||
<li><a class="reference internal" href="#ast-extensions">AST extensions</a></li>
|
||
<li><a class="reference internal" href="#hygiene-and-debugging">Hygiene and debugging</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#examples">Examples</a><ul>
|
||
<li><a class="reference internal" href="#compile-time-checked-data-structures">Compile-time-checked data structures</a></li>
|
||
<li><a class="reference internal" href="#domain-specific-extensions">Domain-specific extensions</a></li>
|
||
<li><a class="reference internal" href="#compilers">Compilers</a></li>
|
||
<li><a class="reference internal" href="#matching-symbolic-expressions">Matching symbolic expressions</a></li>
|
||
<li><a class="reference internal" href="#zero-cost-markers-and-annotations">Zero-cost markers and annotations</a></li>
|
||
<li><a class="reference internal" href="#prototyping-language-extensions">Prototyping language extensions</a><ul>
|
||
<li><a class="reference internal" href="#f-strings">f-strings:</a></li>
|
||
<li><a class="reference internal" href="#try-finally-statement">Try finally statement:</a></li>
|
||
<li><a class="reference internal" href="#with-statement">With statement:</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#macro-definition-macros">Macro definition macros</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#performance-implications">Performance Implications</a></li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a><ul>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0638.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |