mirror of https://github.com/python/peps
487 lines
39 KiB
HTML
487 lines
39 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 521 – Managing global context via ‘with’ blocks in generators and coroutines | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0521/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 521 – Managing global context via ‘with’ blocks in generators and coroutines | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0521/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 521</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 521 – Managing global context via ‘with’ blocks in generators and coroutines</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Nathaniel J. Smith <njs at pobox.com></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Removed from consideration by sponsor or authors">Withdrawn</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">27-Apr-2015</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.6</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">29-Apr-2015</dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#pep-withdrawal">PEP Withdrawal</a></li>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#nested-blocks">Nested blocks</a></li>
|
||
<li><a class="reference internal" href="#other-changes">Other changes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a><ul>
|
||
<li><a class="reference internal" href="#alternative-approaches">Alternative approaches</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards compatibility</a></li>
|
||
<li><a class="reference internal" href="#interaction-with-pep-492">Interaction with PEP 492</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="pep-withdrawal">
|
||
<h2><a class="toc-backref" href="#pep-withdrawal" role="doc-backlink">PEP Withdrawal</a></h2>
|
||
<p>Withdrawn in favor of <a class="pep reference internal" href="../pep-0567/" title="PEP 567 – Context Variables">PEP 567</a>.</p>
|
||
</section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>While we generally try to avoid global state when possible, there
|
||
nonetheless exist a number of situations where it is agreed to be the
|
||
best approach. In Python, a standard pattern for handling such cases
|
||
is to store the global state in global or thread-local storage, and
|
||
then use <code class="docutils literal notranslate"><span class="pre">with</span></code> blocks to limit modifications of this global state
|
||
to a single dynamic scope. Examples where this pattern is used include
|
||
the standard library’s <code class="docutils literal notranslate"><span class="pre">warnings.catch_warnings</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">decimal.localcontext</span></code>, NumPy’s <code class="docutils literal notranslate"><span class="pre">numpy.errstate</span></code> (which exposes
|
||
the error-handling settings provided by the IEEE 754 floating point
|
||
standard), and the handling of logging context or HTTP request context
|
||
in many server application frameworks.</p>
|
||
<p>However, there is currently no ergonomic way to manage such local
|
||
changes to global state when writing a generator or coroutine. For
|
||
example, this code:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">f</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span>
|
||
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">g</span><span class="p">():</span>
|
||
<span class="k">yield</span> <span class="n">x</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>may or may not successfully catch warnings raised by <code class="docutils literal notranslate"><span class="pre">g()</span></code>, and may
|
||
or may not inadvertently swallow warnings triggered elsewhere in the
|
||
code. The context manager, which was intended to apply only to <code class="docutils literal notranslate"><span class="pre">f</span></code>
|
||
and its callees, ends up having a dynamic scope that encompasses
|
||
arbitrary and unpredictable parts of its call<strong>ers</strong>. This problem
|
||
becomes particularly acute when writing asynchronous code, where
|
||
essentially all functions become coroutines.</p>
|
||
<p>Here, we propose to solve this problem by notifying context managers
|
||
whenever execution is suspended or resumed within their scope,
|
||
allowing them to restrict their effects appropriately.</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<p>Two new, optional, methods are added to the context manager protocol:
|
||
<code class="docutils literal notranslate"><span class="pre">__suspend__</span></code> and <code class="docutils literal notranslate"><span class="pre">__resume__</span></code>. If present, these methods will be
|
||
called whenever a frame’s execution is suspended or resumed from
|
||
within the context of the <code class="docutils literal notranslate"><span class="pre">with</span></code> block.</p>
|
||
<p>More formally, consider the following code:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">with</span> <span class="n">EXPR</span> <span class="k">as</span> <span class="n">VAR</span><span class="p">:</span>
|
||
<span class="n">PARTIAL</span><span class="o">-</span><span class="n">BLOCK</span><span class="o">-</span><span class="mi">1</span>
|
||
<span class="n">f</span><span class="p">((</span><span class="k">yield</span> <span class="n">foo</span><span class="p">))</span>
|
||
<span class="n">PARTIAL</span><span class="o">-</span><span class="n">BLOCK</span><span class="o">-</span><span class="mi">2</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Currently this is equivalent to the following code (copied from <a class="pep reference internal" href="../pep-0343/" title="PEP 343 – The “with” Statement">PEP 343</a>):</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">mgr</span> <span class="o">=</span> <span class="p">(</span><span class="n">EXPR</span><span class="p">)</span>
|
||
<span class="n">exit</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span><span class="o">.</span><span class="fm">__exit__</span> <span class="c1"># Not calling it yet</span>
|
||
<span class="n">value</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span><span class="o">.</span><span class="fm">__enter__</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span>
|
||
<span class="n">exc</span> <span class="o">=</span> <span class="kc">True</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">VAR</span> <span class="o">=</span> <span class="n">value</span> <span class="c1"># Only if "as VAR" is present</span>
|
||
<span class="n">PARTIAL</span><span class="o">-</span><span class="n">BLOCK</span><span class="o">-</span><span class="mi">1</span>
|
||
<span class="n">f</span><span class="p">((</span><span class="k">yield</span> <span class="n">foo</span><span class="p">))</span>
|
||
<span class="n">PARTIAL</span><span class="o">-</span><span class="n">BLOCK</span><span class="o">-</span><span class="mi">2</span>
|
||
<span class="k">except</span><span class="p">:</span>
|
||
<span class="n">exc</span> <span class="o">=</span> <span class="kc">False</span>
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">exit</span><span class="p">(</span><span class="n">mgr</span><span class="p">,</span> <span class="o">*</span><span class="n">sys</span><span class="o">.</span><span class="n">exc_info</span><span class="p">()):</span>
|
||
<span class="k">raise</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="k">if</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="n">exit</span><span class="p">(</span><span class="n">mgr</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This PEP proposes to modify <code class="docutils literal notranslate"><span class="pre">with</span></code> block handling to instead become:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">mgr</span> <span class="o">=</span> <span class="p">(</span><span class="n">EXPR</span><span class="p">)</span>
|
||
<span class="n">exit</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span><span class="o">.</span><span class="fm">__exit__</span> <span class="c1"># Not calling it yet</span>
|
||
<span class="c1">### --- NEW STUFF ---</span>
|
||
<span class="k">if</span> <span class="n">the_block_contains_yield_points</span><span class="p">:</span> <span class="c1"># known statically at compile time</span>
|
||
<span class="n">suspend</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">mgr</span><span class="p">),</span> <span class="s2">"__suspend__"</span><span class="p">,</span> <span class="k">lambda</span><span class="p">:</span> <span class="kc">None</span><span class="p">)</span>
|
||
<span class="n">resume</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">mgr</span><span class="p">),</span> <span class="s2">"__resume__"</span><span class="p">,</span> <span class="k">lambda</span><span class="p">:</span> <span class="kc">None</span><span class="p">)</span>
|
||
<span class="c1">### --- END OF NEW STUFF ---</span>
|
||
<span class="n">value</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span><span class="o">.</span><span class="fm">__enter__</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span>
|
||
<span class="n">exc</span> <span class="o">=</span> <span class="kc">True</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">VAR</span> <span class="o">=</span> <span class="n">value</span> <span class="c1"># Only if "as VAR" is present</span>
|
||
<span class="n">PARTIAL</span><span class="o">-</span><span class="n">BLOCK</span><span class="o">-</span><span class="mi">1</span>
|
||
<span class="c1">### --- NEW STUFF ---</span>
|
||
<span class="n">suspend</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span>
|
||
<span class="n">tmp</span> <span class="o">=</span> <span class="k">yield</span> <span class="n">foo</span>
|
||
<span class="n">resume</span><span class="p">(</span><span class="n">mgr</span><span class="p">)</span>
|
||
<span class="n">f</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span>
|
||
<span class="c1">### --- END OF NEW STUFF ---</span>
|
||
<span class="n">PARTIAL</span><span class="o">-</span><span class="n">BLOCK</span><span class="o">-</span><span class="mi">2</span>
|
||
<span class="k">except</span><span class="p">:</span>
|
||
<span class="n">exc</span> <span class="o">=</span> <span class="kc">False</span>
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">exit</span><span class="p">(</span><span class="n">mgr</span><span class="p">,</span> <span class="o">*</span><span class="n">sys</span><span class="o">.</span><span class="n">exc_info</span><span class="p">()):</span>
|
||
<span class="k">raise</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="k">if</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="n">exit</span><span class="p">(</span><span class="n">mgr</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Analogous suspend/resume calls are also wrapped around the <code class="docutils literal notranslate"><span class="pre">yield</span></code>
|
||
points embedded inside the <code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code>, <code class="docutils literal notranslate"><span class="pre">await</span></code>, <code class="docutils literal notranslate"><span class="pre">async</span> <span class="pre">with</span></code>,
|
||
and <code class="docutils literal notranslate"><span class="pre">async</span> <span class="pre">for</span></code> constructs.</p>
|
||
<section id="nested-blocks">
|
||
<h3><a class="toc-backref" href="#nested-blocks" role="doc-backlink">Nested blocks</a></h3>
|
||
<p>Given this code:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">f</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">OUTER</span><span class="p">:</span>
|
||
<span class="k">with</span> <span class="n">INNER</span><span class="p">:</span>
|
||
<span class="k">yield</span> <span class="n">VALUE</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>then we perform the following operations in the following sequence:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">INNER</span><span class="o">.</span><span class="n">__suspend__</span><span class="p">()</span>
|
||
<span class="n">OUTER</span><span class="o">.</span><span class="n">__suspend__</span><span class="p">()</span>
|
||
<span class="k">yield</span> <span class="n">VALUE</span>
|
||
<span class="n">OUTER</span><span class="o">.</span><span class="n">__resume__</span><span class="p">()</span>
|
||
<span class="n">INNER</span><span class="o">.</span><span class="n">__resume__</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Note that this ensures that the following is a valid refactoring:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">f</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">OUTER</span><span class="p">:</span>
|
||
<span class="k">yield from</span> <span class="n">g</span><span class="p">()</span>
|
||
|
||
<span class="k">def</span> <span class="nf">g</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">INNER</span>
|
||
<span class="k">yield</span> <span class="n">VALUE</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Similarly, <code class="docutils literal notranslate"><span class="pre">with</span></code> statements with multiple context managers suspend
|
||
from right to left, and resume from left to right.</p>
|
||
</section>
|
||
<section id="other-changes">
|
||
<h3><a class="toc-backref" href="#other-changes" role="doc-backlink">Other changes</a></h3>
|
||
<p>Appropriate <code class="docutils literal notranslate"><span class="pre">__suspend__</span></code> and <code class="docutils literal notranslate"><span class="pre">__resume__</span></code> methods are added to
|
||
<code class="docutils literal notranslate"><span class="pre">warnings.catch_warnings</span></code> and <code class="docutils literal notranslate"><span class="pre">decimal.localcontext</span></code>.</p>
|
||
</section>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>In the abstract, we gave an example of plausible but incorrect code:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">f</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span>
|
||
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">g</span><span class="p">():</span>
|
||
<span class="k">yield</span> <span class="n">x</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>To make this correct in current Python, we need to instead write
|
||
something like:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">f</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span>
|
||
<span class="n">it</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">g</span><span class="p">())</span>
|
||
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
|
||
<span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">it</span><span class="p">)</span>
|
||
<span class="k">except</span> <span class="ne">StopIteration</span><span class="p">:</span>
|
||
<span class="k">break</span>
|
||
<span class="k">yield</span> <span class="n">x</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>OTOH, if this PEP is accepted then the original code will become
|
||
correct as-is. Or if this isn’t convincing, then here’s another
|
||
example of broken code; fixing it requires even greater gyrations, and
|
||
these are left as an exercise for the reader:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">async</span> <span class="k">def</span> <span class="nf">test_foo_emits_warning</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">(</span><span class="n">record</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="k">as</span> <span class="n">w</span><span class="p">:</span>
|
||
<span class="k">await</span> <span class="n">foo</span><span class="p">()</span>
|
||
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">w</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span>
|
||
<span class="k">assert</span> <span class="s2">"xyzzy"</span> <span class="ow">in</span> <span class="n">w</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">message</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>And notice that this last example isn’t artificial at all – this is
|
||
exactly how you write a test that an async/await-using coroutine
|
||
correctly raises a warning. Similar issues arise for pretty much any
|
||
use of <code class="docutils literal notranslate"><span class="pre">warnings.catch_warnings</span></code>, <code class="docutils literal notranslate"><span class="pre">decimal.localcontext</span></code>, or
|
||
<code class="docutils literal notranslate"><span class="pre">numpy.errstate</span></code> in async/await-using code. So there’s clearly a
|
||
real problem to solve here, and the growing prominence of async code
|
||
makes it increasingly urgent.</p>
|
||
<section id="alternative-approaches">
|
||
<h3><a class="toc-backref" href="#alternative-approaches" role="doc-backlink">Alternative approaches</a></h3>
|
||
<p>The main alternative that has been proposed is to create some kind of
|
||
“task-local storage”, analogous to “thread-local storage”
|
||
<a class="footnote-reference brackets" href="#yury-task-local-proposal" id="id1">[1]</a>. In essence, the idea would be that the
|
||
event loop would take care to allocate a new “task namespace” for each
|
||
task it schedules, and provide an API to at any given time fetch the
|
||
namespace corresponding to the currently executing task. While there
|
||
are many details to be worked out <a class="footnote-reference brackets" href="#task-local-challenges" id="id2">[2]</a>, the basic
|
||
idea seems doable, and it is an especially natural way to handle the
|
||
kind of global context that arises at the top-level of async
|
||
application frameworks (e.g., setting up context objects in a web
|
||
framework). But it also has a number of flaws:</p>
|
||
<ul>
|
||
<li>It only solves the problem of managing global state for coroutines
|
||
that <code class="docutils literal notranslate"><span class="pre">yield</span></code> back to an asynchronous event loop. But there
|
||
actually isn’t anything about this problem that’s specific to
|
||
asyncio – as shown in the examples above, simple generators run
|
||
into exactly the same issue.</li>
|
||
<li>It creates an unnecessary coupling between event loops and code that
|
||
needs to manage global state. Obviously an async web framework needs
|
||
to interact with some event loop API anyway, so it’s not a big deal
|
||
in that case. But it’s weird that <code class="docutils literal notranslate"><span class="pre">warnings</span></code> or <code class="docutils literal notranslate"><span class="pre">decimal</span></code> or
|
||
NumPy should have to call into an async library’s API to access
|
||
their internal state when they themselves involve no async code.
|
||
Worse, since there are multiple event loop APIs in common use, it
|
||
isn’t clear how to choose which to integrate with. (This could be
|
||
somewhat mitigated by CPython providing a standard API for creating
|
||
and switching “task-local domains” that asyncio, Twisted, tornado,
|
||
etc. could then work with.)</li>
|
||
<li>It’s not at all clear that this can be made acceptably fast. NumPy
|
||
has to check the floating point error settings on every single
|
||
arithmetic operation. Checking a piece of data in thread-local
|
||
storage is absurdly quick, because modern platforms have put massive
|
||
resources into optimizing this case (e.g. dedicating a CPU register
|
||
for this purpose); calling a method on an event loop to fetch a
|
||
handle to a namespace and then doing lookup in that namespace is
|
||
much slower.<p>More importantly, this extra cost would be paid on <em>every</em> access to
|
||
the global data, even for programs which are not otherwise using an
|
||
event loop at all. This PEP’s proposal, by contrast, only affects
|
||
code that actually mixes <code class="docutils literal notranslate"><span class="pre">with</span></code> blocks and <code class="docutils literal notranslate"><span class="pre">yield</span></code> statements,
|
||
meaning that the users who experience the costs are the same users
|
||
who also reap the benefits.</p>
|
||
</li>
|
||
</ul>
|
||
<p>On the other hand, such tight integration between task context and the
|
||
event loop does potentially allow other features that are beyond the
|
||
scope of the current proposal. For example, an event loop could note
|
||
which task namespace was in effect when a task called <code class="docutils literal notranslate"><span class="pre">call_soon</span></code>,
|
||
and arrange that the callback when run would have access to the same
|
||
task namespace. Whether this is useful, or even well-defined in the
|
||
case of cross-thread calls (what does it mean to have task-local
|
||
storage accessed from two threads simultaneously?), is left as a
|
||
puzzle for event loop implementors to ponder – nothing in this
|
||
proposal rules out such enhancements as well. It does seem though
|
||
that such features would be useful primarily for state that already
|
||
has a tight integration with the event loop – while we might want a
|
||
request id to be preserved across <code class="docutils literal notranslate"><span class="pre">call_soon</span></code>, most people would not
|
||
expect:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span>
|
||
<span class="n">loop</span><span class="o">.</span><span class="n">call_soon</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>to result in <code class="docutils literal notranslate"><span class="pre">f</span></code> being run with warnings disabled, which would be
|
||
the result if <code class="docutils literal notranslate"><span class="pre">call_soon</span></code> preserved global context in general. It’s
|
||
also unclear how this would even work given that the warnings context
|
||
manager <code class="docutils literal notranslate"><span class="pre">__exit__</span></code> would be called before <code class="docutils literal notranslate"><span class="pre">f</span></code>.</p>
|
||
<p>So this PEP takes the position that <code class="docutils literal notranslate"><span class="pre">__suspend__</span></code>/<code class="docutils literal notranslate"><span class="pre">__resume__</span></code>
|
||
and “task-local storage” are two complementary tools that are both
|
||
useful in different circumstances.</p>
|
||
</section>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards compatibility</a></h2>
|
||
<p>Because <code class="docutils literal notranslate"><span class="pre">__suspend__</span></code> and <code class="docutils literal notranslate"><span class="pre">__resume__</span></code> are optional and default to
|
||
no-ops, all existing context managers continue to work exactly as
|
||
before.</p>
|
||
<p>Speed-wise, this proposal adds additional overhead when entering a
|
||
<code class="docutils literal notranslate"><span class="pre">with</span></code> block (where we must now check for the additional methods;
|
||
failed attribute lookup in CPython is rather slow, since it involves
|
||
allocating an <code class="docutils literal notranslate"><span class="pre">AttributeError</span></code>), and additional overhead at
|
||
suspension points. Since the position of <code class="docutils literal notranslate"><span class="pre">with</span></code> blocks and
|
||
suspension points is known statically, the compiler can
|
||
straightforwardly optimize away this overhead in all cases except
|
||
where one actually has a <code class="docutils literal notranslate"><span class="pre">yield</span></code> inside a <code class="docutils literal notranslate"><span class="pre">with</span></code>. Furthermore,
|
||
because we only do attribute checks for <code class="docutils literal notranslate"><span class="pre">__suspend__</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">__resume__</span></code> once at the start of a <code class="docutils literal notranslate"><span class="pre">with</span></code> block, when these
|
||
attributes are undefined then the per-yield overhead can be optimized
|
||
down to a single C-level <code class="docutils literal notranslate"><span class="pre">if</span> <span class="pre">(frame->needs_suspend_resume_calls)</span> <span class="pre">{</span>
|
||
<span class="pre">...</span> <span class="pre">}</span></code>. Therefore, we expect the overall overhead to be negligible.</p>
|
||
</section>
|
||
<section id="interaction-with-pep-492">
|
||
<h2><a class="toc-backref" href="#interaction-with-pep-492" role="doc-backlink">Interaction with PEP 492</a></h2>
|
||
<p><a class="pep reference internal" href="../pep-0492/" title="PEP 492 – Coroutines with async and await syntax">PEP 492</a> added new asynchronous context managers, which are like
|
||
regular context managers, but instead of having regular methods
|
||
<code class="docutils literal notranslate"><span class="pre">__enter__</span></code> and <code class="docutils literal notranslate"><span class="pre">__exit__</span></code> they have coroutine methods
|
||
<code class="docutils literal notranslate"><span class="pre">__aenter__</span></code> and <code class="docutils literal notranslate"><span class="pre">__aexit__</span></code>.</p>
|
||
<p>Following this pattern, one might expect this proposal to add
|
||
<code class="docutils literal notranslate"><span class="pre">__asuspend__</span></code> and <code class="docutils literal notranslate"><span class="pre">__aresume__</span></code> coroutine methods. But this
|
||
doesn’t make much sense, since the whole point is that <code class="docutils literal notranslate"><span class="pre">__suspend__</span></code>
|
||
should be called before yielding our thread of execution and allowing
|
||
other code to run. The only thing we accomplish by making
|
||
<code class="docutils literal notranslate"><span class="pre">__asuspend__</span></code> a coroutine is to make it possible for
|
||
<code class="docutils literal notranslate"><span class="pre">__asuspend__</span></code> itself to yield. So either we need to recursively
|
||
call <code class="docutils literal notranslate"><span class="pre">__asuspend__</span></code> from inside <code class="docutils literal notranslate"><span class="pre">__asuspend__</span></code>, or else we need to
|
||
give up and allow these yields to happen without calling the suspend
|
||
callback; either way it defeats the whole point.</p>
|
||
<p>Well, with one exception: one possible pattern for coroutine code is
|
||
to call <code class="docutils literal notranslate"><span class="pre">yield</span></code> in order to communicate with the coroutine runner,
|
||
but without actually suspending their execution (i.e., the coroutine
|
||
might know that the coroutine runner will resume them immediately
|
||
after processing the <code class="docutils literal notranslate"><span class="pre">yield</span></code>ed message). An example of this is the
|
||
<code class="docutils literal notranslate"><span class="pre">curio.timeout_after</span></code> async context manager, which yields a special
|
||
<code class="docutils literal notranslate"><span class="pre">set_timeout</span></code> message to the curio kernel, and then the kernel
|
||
immediately (synchronously) resumes the coroutine which sent the
|
||
message. And from the user point of view, this timeout value acts just
|
||
like the kinds of global variables that motivated this PEP. But, there
|
||
is a crucal difference: this kind of async context manager is, by
|
||
definition, tightly integrated with the coroutine runner. So, the
|
||
coroutine runner can take over responsibility for keeping track of
|
||
which timeouts apply to which coroutines without any need for this PEP
|
||
at all (and this is indeed how curio.timeout_after works).</p>
|
||
<p>That leaves two reasonable approaches to handling async context managers:</p>
|
||
<ol class="arabic simple">
|
||
<li>Add plain <code class="docutils literal notranslate"><span class="pre">__suspend__</span></code> and <code class="docutils literal notranslate"><span class="pre">__resume__</span></code> methods.</li>
|
||
<li>Leave async context managers alone for now until we have more
|
||
experience with them.</li>
|
||
</ol>
|
||
<p>Either seems plausible, so out of laziness / <a class="reference external" href="http://martinfowler.com/bliki/Yagni.html">YAGNI</a> this PEP tentatively
|
||
proposes to stick with option (2).</p>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="yury-task-local-proposal" role="doc-footnote">
|
||
<dt class="label" id="yury-task-local-proposal">[<a href="#id1">1</a>]</dt>
|
||
<dd><a class="reference external" href="https://groups.google.com/forum/#!topic/python-tulip/zix5HQxtElg">https://groups.google.com/forum/#!topic/python-tulip/zix5HQxtElg</a>
|
||
<a class="reference external" href="https://github.com/python/asyncio/issues/165">https://github.com/python/asyncio/issues/165</a></aside>
|
||
<aside class="footnote brackets" id="task-local-challenges" role="doc-footnote">
|
||
<dt class="label" id="task-local-challenges">[<a href="#id2">2</a>]</dt>
|
||
<dd>For example, we would have to decide
|
||
whether there is a single task-local namespace shared by all users
|
||
(in which case we need a way for multiple third-party libraries to
|
||
adjudicate access to this namespace), or else if there are multiple
|
||
task-local namespaces, then we need some mechanism for each library
|
||
to arrange for their task-local namespaces to be created and
|
||
destroyed at appropriate moments. The preliminary patch linked
|
||
from the github issue above doesn’t seem to provide any mechanism
|
||
for such lifecycle management.</aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0521.rst">https://github.com/python/peps/blob/main/peps/pep-0521.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0521.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#pep-withdrawal">PEP Withdrawal</a></li>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#nested-blocks">Nested blocks</a></li>
|
||
<li><a class="reference internal" href="#other-changes">Other changes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a><ul>
|
||
<li><a class="reference internal" href="#alternative-approaches">Alternative approaches</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards compatibility</a></li>
|
||
<li><a class="reference internal" href="#interaction-with-pep-492">Interaction with PEP 492</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0521.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |