mirror of https://github.com/python/peps
1835 lines
159 KiB
HTML
1835 lines
159 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 550 – Execution Context | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0550/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 550 – Execution Context | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0550/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 550</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 550 – Execution Context</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Yury Selivanov <yury at edgedb.com>,
|
||
Elvis Pranskevichus <elvis at edgedb.com></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Removed from consideration by sponsor or authors">Withdrawn</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">11-Aug-2017</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.7</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">11-Aug-2017, 15-Aug-2017, 18-Aug-2017, 25-Aug-2017,
|
||
01-Sep-2017</dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#pep-status">PEP Status</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#goals">Goals</a></li>
|
||
<li><a class="reference internal" href="#high-level-specification">High-Level Specification</a><ul>
|
||
<li><a class="reference internal" href="#regular-single-threaded-code">Regular Single-threaded Code</a></li>
|
||
<li><a class="reference internal" href="#multithreaded-code">Multithreaded Code</a></li>
|
||
<li><a class="reference internal" href="#generators">Generators</a></li>
|
||
<li><a class="reference internal" href="#coroutines-and-asynchronous-tasks">Coroutines and Asynchronous Tasks</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#detailed-specification">Detailed Specification</a><ul>
|
||
<li><a class="reference internal" href="#id5">Generators</a></li>
|
||
<li><a class="reference internal" href="#contextlib-contextmanager">contextlib.contextmanager</a></li>
|
||
<li><a class="reference internal" href="#enumerating-context-vars">Enumerating context vars</a></li>
|
||
<li><a class="reference internal" href="#coroutines">coroutines</a></li>
|
||
<li><a class="reference internal" href="#asynchronous-generators">Asynchronous Generators</a></li>
|
||
<li><a class="reference internal" href="#asyncio">asyncio</a></li>
|
||
<li><a class="reference internal" href="#generators-transformed-into-iterators">Generators Transformed into Iterators</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a><ul>
|
||
<li><a class="reference internal" href="#logical-context">Logical Context</a></li>
|
||
<li><a class="reference internal" href="#context-variables">Context Variables</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#performance-considerations">Performance Considerations</a></li>
|
||
<li><a class="reference internal" href="#summary-of-the-new-apis">Summary of the New APIs</a><ul>
|
||
<li><a class="reference internal" href="#python">Python</a></li>
|
||
<li><a class="reference internal" href="#c-api">C API</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#design-considerations">Design Considerations</a><ul>
|
||
<li><a class="reference internal" href="#should-yield-from-leak-context-changes">Should “yield from” leak context changes?</a></li>
|
||
<li><a class="reference internal" href="#should-pythreadstate-getdict-use-the-execution-context">Should <code class="docutils literal notranslate"><span class="pre">PyThreadState_GetDict()</span></code> use the execution context?</a></li>
|
||
<li><a class="reference internal" href="#pep-521">PEP 521</a></li>
|
||
<li><a class="reference internal" href="#can-execution-context-be-implemented-without-modifying-cpython">Can Execution Context be implemented without modifying CPython?</a></li>
|
||
<li><a class="reference internal" href="#should-we-update-sys-displayhook-and-other-apis-to-use-ec">Should we update sys.displayhook and other APIs to use EC?</a></li>
|
||
<li><a class="reference internal" href="#greenlets">Greenlets</a></li>
|
||
<li><a class="reference internal" href="#context-manager-as-the-interface-for-modifications">Context manager as the interface for modifications</a></li>
|
||
<li><a class="reference internal" href="#setting-and-restoring-context-variables">Setting and restoring context variables</a></li>
|
||
<li><a class="reference internal" href="#alternative-designs-for-contextvar-api">Alternative Designs for ContextVar API</a><ul>
|
||
<li><a class="reference internal" href="#logical-context-with-stacked-values">Logical Context with stacked values</a></li>
|
||
<li><a class="reference internal" href="#contextvar-set-reset">ContextVar “set/reset”</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#replication-of-threading-local-interface">Replication of threading.local() interface</a></li>
|
||
<li><a class="reference internal" href="#coroutines-not-leaking-context-changes-by-default">Coroutines not leaking context changes by default</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#appendix-hamt-performance-analysis">Appendix: HAMT Performance Analysis</a></li>
|
||
<li><a class="reference internal" href="#acknowledgments">Acknowledgments</a></li>
|
||
<li><a class="reference internal" href="#version-history">Version History</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP adds a new generic mechanism of ensuring consistent access
|
||
to non-local state in the context of out-of-order execution, such
|
||
as in Python generators and coroutines.</p>
|
||
<p>Thread-local storage, such as <code class="docutils literal notranslate"><span class="pre">threading.local()</span></code>, is inadequate for
|
||
programs that execute concurrently in the same OS thread. This PEP
|
||
proposes a solution to this problem.</p>
|
||
</section>
|
||
<section id="pep-status">
|
||
<h2><a class="toc-backref" href="#pep-status" role="doc-backlink">PEP Status</a></h2>
|
||
<p>Due to its breadth and the lack of general consensus on some aspects, this
|
||
PEP has been withdrawn and superseded by a simpler <a class="pep reference internal" href="../pep-0567/" title="PEP 567 – Context Variables">PEP 567</a>, which has
|
||
been accepted and included in Python 3.7.</p>
|
||
<p><a class="pep reference internal" href="../pep-0567/" title="PEP 567 – Context Variables">PEP 567</a> implements the same core idea, but limits the ContextVar support
|
||
to asynchronous tasks while leaving the generator behavior untouched.
|
||
The latter may be revisited in a future PEP.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Prior to the advent of asynchronous programming in Python, programs
|
||
used OS threads to achieve concurrency. The need for thread-specific
|
||
state was solved by <code class="docutils literal notranslate"><span class="pre">threading.local()</span></code> and its C-API equivalent,
|
||
<code class="docutils literal notranslate"><span class="pre">PyThreadState_GetDict()</span></code>.</p>
|
||
<p>A few examples of where Thread-local storage (TLS) is commonly
|
||
relied upon:</p>
|
||
<ul class="simple">
|
||
<li>Context managers like decimal contexts, <code class="docutils literal notranslate"><span class="pre">numpy.errstate</span></code>,
|
||
and <code class="docutils literal notranslate"><span class="pre">warnings.catch_warnings</span></code>.</li>
|
||
<li>Request-related data, such as security tokens and request
|
||
data in web applications, language context for <code class="docutils literal notranslate"><span class="pre">gettext</span></code> etc.</li>
|
||
<li>Profiling, tracing, and logging in large code bases.</li>
|
||
</ul>
|
||
<p>Unfortunately, TLS does not work well for programs which execute
|
||
concurrently in a single thread. A Python generator is the simplest
|
||
example of a concurrent program. Consider the following:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">fractions</span><span class="p">(</span><span class="n">precision</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
|
||
<span class="k">with</span> <span class="n">decimal</span><span class="o">.</span><span class="n">localcontext</span><span class="p">()</span> <span class="k">as</span> <span class="n">ctx</span><span class="p">:</span>
|
||
<span class="n">ctx</span><span class="o">.</span><span class="n">prec</span> <span class="o">=</span> <span class="n">precision</span>
|
||
<span class="k">yield</span> <span class="n">Decimal</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">/</span> <span class="n">Decimal</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="k">yield</span> <span class="n">Decimal</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">/</span> <span class="n">Decimal</span><span class="p">(</span><span class="n">y</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">g1</span> <span class="o">=</span> <span class="n">fractions</span><span class="p">(</span><span class="n">precision</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">x</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">g2</span> <span class="o">=</span> <span class="n">fractions</span><span class="p">(</span><span class="n">precision</span><span class="o">=</span><span class="mi">6</span><span class="p">,</span> <span class="n">x</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="n">items</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">g1</span><span class="p">,</span> <span class="n">g2</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The intuitively expected value of <code class="docutils literal notranslate"><span class="pre">items</span></code> is:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[(</span><span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.33'</span><span class="p">),</span> <span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.666667'</span><span class="p">)),</span>
|
||
<span class="p">(</span><span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.11'</span><span class="p">),</span> <span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.222222'</span><span class="p">))]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Rather surprisingly, the actual result is:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[(</span><span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.33'</span><span class="p">),</span> <span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.666667'</span><span class="p">)),</span>
|
||
<span class="p">(</span><span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.111111'</span><span class="p">),</span> <span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.222222'</span><span class="p">))]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This is because implicit Decimal context is stored as a thread-local,
|
||
so concurrent iteration of the <code class="docutils literal notranslate"><span class="pre">fractions()</span></code> generator would
|
||
corrupt the state. For Decimal, specifically, the only current
|
||
workaround is to use explicit context method calls for all arithmetic
|
||
operations <a class="footnote-reference brackets" href="#id49" id="id1">[28]</a>. Arguably, this defeats the usefulness of overloaded
|
||
operators and makes even simple formulas hard to read and write.</p>
|
||
<p>Coroutines are another class of Python code where TLS unreliability
|
||
is a significant issue.</p>
|
||
<p>The inadequacy of TLS in asynchronous code has lead to the
|
||
proliferation of ad-hoc solutions, which are limited in scope and
|
||
do not support all required use cases.</p>
|
||
<p>The current status quo is that any library (including the standard
|
||
library), which relies on TLS, is likely to be broken when used in
|
||
asynchronous code or with generators (see <a class="footnote-reference brackets" href="#id31" id="id2">[3]</a> as an example issue.)</p>
|
||
<p>Some languages, that support coroutines or generators, recommend
|
||
passing the context manually as an argument to every function, see
|
||
<a class="footnote-reference brackets" href="#id29" id="id3">[1]</a> for an example. This approach, however, has limited use for
|
||
Python, where there is a large ecosystem that was built to work with
|
||
a TLS-like context. Furthermore, libraries like <code class="docutils literal notranslate"><span class="pre">decimal</span></code> or
|
||
<code class="docutils literal notranslate"><span class="pre">numpy</span></code> rely on context implicitly in overloaded operator
|
||
implementations.</p>
|
||
<p>The .NET runtime, which has support for async/await, has a generic
|
||
solution for this problem, called <code class="docutils literal notranslate"><span class="pre">ExecutionContext</span></code> (see <a class="footnote-reference brackets" href="#id30" id="id4">[2]</a>).</p>
|
||
</section>
|
||
<section id="goals">
|
||
<h2><a class="toc-backref" href="#goals" role="doc-backlink">Goals</a></h2>
|
||
<p>The goal of this PEP is to provide a more reliable
|
||
<code class="docutils literal notranslate"><span class="pre">threading.local()</span></code> alternative, which:</p>
|
||
<ul class="simple">
|
||
<li>provides the mechanism and the API to fix non-local state issues
|
||
with coroutines and generators;</li>
|
||
<li>implements TLS-like semantics for synchronous code, so that
|
||
users like <code class="docutils literal notranslate"><span class="pre">decimal</span></code> and <code class="docutils literal notranslate"><span class="pre">numpy</span></code> can switch to the new
|
||
mechanism with minimal risk of breaking backwards compatibility;</li>
|
||
<li>has no or negligible performance impact on the existing code or
|
||
the code that will be using the new mechanism, including
|
||
C extensions.</li>
|
||
</ul>
|
||
</section>
|
||
<section id="high-level-specification">
|
||
<h2><a class="toc-backref" href="#high-level-specification" role="doc-backlink">High-Level Specification</a></h2>
|
||
<p>The full specification of this PEP is broken down into three parts:</p>
|
||
<ul class="simple">
|
||
<li>High-Level Specification (this section): the description of the
|
||
overall solution. We show how it applies to generators and
|
||
coroutines in user code, without delving into implementation
|
||
details.</li>
|
||
<li>Detailed Specification: the complete description of new concepts,
|
||
APIs, and related changes to the standard library.</li>
|
||
<li>Implementation Details: the description and analysis of data
|
||
structures and algorithms used to implement this PEP, as well as
|
||
the necessary changes to CPython.</li>
|
||
</ul>
|
||
<p>For the purpose of this section, we define <em>execution context</em> as an
|
||
opaque container of non-local state that allows consistent access to
|
||
its contents in the concurrent execution environment.</p>
|
||
<p>A <em>context variable</em> is an object representing a value in the
|
||
execution context. A call to <code class="docutils literal notranslate"><span class="pre">contextvars.ContextVar(name)</span></code>
|
||
creates a new context variable object. A context variable object has
|
||
three methods:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">get()</span></code>: returns the value of the variable in the current
|
||
execution context;</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">set(value)</span></code>: sets the value of the variable in the current
|
||
execution context;</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">delete()</span></code>: can be used for restoring variable state, it’s
|
||
purpose and semantics are explained in
|
||
<a class="reference internal" href="#setting-and-restoring-context-variables">Setting and restoring context variables</a>.</li>
|
||
</ul>
|
||
<section id="regular-single-threaded-code">
|
||
<h3><a class="toc-backref" href="#regular-single-threaded-code" role="doc-backlink">Regular Single-threaded Code</a></h3>
|
||
<p>In regular, single-threaded code that doesn’t involve generators or
|
||
coroutines, context variables behave like globals:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">sub</span><span class="p">():</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main'</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'sub'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
<span class="n">sub</span><span class="p">()</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'sub'</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="multithreaded-code">
|
||
<h3><a class="toc-backref" href="#multithreaded-code" role="doc-backlink">Multithreaded Code</a></h3>
|
||
<p>In multithreaded code, context variables behave like thread locals:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">sub</span><span class="p">():</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span> <span class="c1"># The execution context is empty</span>
|
||
<span class="c1"># for each new thread.</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'sub'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
|
||
<span class="n">thread</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">sub</span><span class="p">)</span>
|
||
<span class="n">thread</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||
<span class="n">thread</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
|
||
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main'</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="generators">
|
||
<h3><a class="toc-backref" href="#generators" role="doc-backlink">Generators</a></h3>
|
||
<p>Unlike regular function calls, generators can cooperatively yield
|
||
their control of execution to the caller. Furthermore, a generator
|
||
does not control <em>where</em> the execution would continue after it yields.
|
||
It may be resumed from an arbitrary code location.</p>
|
||
<p>For these reasons, the least surprising behaviour of generators is
|
||
as follows:</p>
|
||
<ul class="simple">
|
||
<li>changes to context variables are always local and are not visible
|
||
in the outer context, but are visible to the code called by the
|
||
generator;</li>
|
||
<li>once set in the generator, the context variable is guaranteed not
|
||
to change between iterations;</li>
|
||
<li>changes to context variables in outer context (where the generator
|
||
is being iterated) are visible to the generator, unless these
|
||
variables were also modified inside the generator.</li>
|
||
</ul>
|
||
<p>Let’s review:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var1</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var1'</span><span class="p">)</span>
|
||
<span class="n">var2</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var2'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">gen</span><span class="p">():</span>
|
||
<span class="n">var1</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'gen'</span><span class="p">)</span>
|
||
<span class="k">assert</span> <span class="n">var1</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'gen'</span>
|
||
<span class="k">assert</span> <span class="n">var2</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main'</span>
|
||
<span class="k">yield</span> <span class="mi">1</span>
|
||
|
||
<span class="c1"># Modification to var1 in main() is shielded by</span>
|
||
<span class="c1"># gen()'s local modification.</span>
|
||
<span class="k">assert</span> <span class="n">var1</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'gen'</span>
|
||
|
||
<span class="c1"># But modifications to var2 are visible</span>
|
||
<span class="k">assert</span> <span class="n">var2</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main modified'</span>
|
||
<span class="k">yield</span> <span class="mi">2</span>
|
||
|
||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">g</span> <span class="o">=</span> <span class="n">gen</span><span class="p">()</span>
|
||
|
||
<span class="n">var1</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
<span class="n">var2</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
<span class="nb">next</span><span class="p">(</span><span class="n">g</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Modification of var1 in gen() is not visible.</span>
|
||
<span class="k">assert</span> <span class="n">var1</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main'</span>
|
||
|
||
<span class="n">var1</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main modified'</span><span class="p">)</span>
|
||
<span class="n">var2</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main modified'</span><span class="p">)</span>
|
||
<span class="nb">next</span><span class="p">(</span><span class="n">g</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Now, let’s revisit the decimal precision example from the <a class="reference internal" href="#rationale">Rationale</a>
|
||
section, and see how the execution context can improve the situation:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">decimal</span>
|
||
|
||
<span class="c1"># create a new context var</span>
|
||
<span class="n">decimal_ctx</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'decimal context'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Pre-PEP 550 Decimal relies on TLS for its context.</span>
|
||
<span class="c1"># For illustration purposes, we monkey-patch the decimal</span>
|
||
<span class="c1"># context functions to use the execution context.</span>
|
||
<span class="c1"># A real working fix would need to properly update the</span>
|
||
<span class="c1"># C implementation as well.</span>
|
||
<span class="k">def</span> <span class="nf">patched_setcontext</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
|
||
<span class="n">decimal_ctx</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">patched_getcontext</span><span class="p">():</span>
|
||
<span class="n">ctx</span> <span class="o">=</span> <span class="n">decimal_ctx</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
|
||
<span class="k">if</span> <span class="n">ctx</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">ctx</span> <span class="o">=</span> <span class="n">decimal</span><span class="o">.</span><span class="n">Context</span><span class="p">()</span>
|
||
<span class="n">decimal_ctx</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">ctx</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">ctx</span>
|
||
|
||
<span class="n">decimal</span><span class="o">.</span><span class="n">setcontext</span> <span class="o">=</span> <span class="n">patched_setcontext</span>
|
||
<span class="n">decimal</span><span class="o">.</span><span class="n">getcontext</span> <span class="o">=</span> <span class="n">patched_getcontext</span>
|
||
|
||
<span class="k">def</span> <span class="nf">fractions</span><span class="p">(</span><span class="n">precision</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
|
||
<span class="k">with</span> <span class="n">decimal</span><span class="o">.</span><span class="n">localcontext</span><span class="p">()</span> <span class="k">as</span> <span class="n">ctx</span><span class="p">:</span>
|
||
<span class="n">ctx</span><span class="o">.</span><span class="n">prec</span> <span class="o">=</span> <span class="n">precision</span>
|
||
<span class="k">yield</span> <span class="n">MyDecimal</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">/</span> <span class="n">MyDecimal</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="k">yield</span> <span class="n">MyDecimal</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">/</span> <span class="n">MyDecimal</span><span class="p">(</span><span class="n">y</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">g1</span> <span class="o">=</span> <span class="n">fractions</span><span class="p">(</span><span class="n">precision</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">x</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">g2</span> <span class="o">=</span> <span class="n">fractions</span><span class="p">(</span><span class="n">precision</span><span class="o">=</span><span class="mi">6</span><span class="p">,</span> <span class="n">x</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
|
||
<span class="n">items</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">g1</span><span class="p">,</span> <span class="n">g2</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The value of <code class="docutils literal notranslate"><span class="pre">items</span></code> is:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[(</span><span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.33'</span><span class="p">),</span> <span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.666667'</span><span class="p">)),</span>
|
||
<span class="p">(</span><span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.11'</span><span class="p">),</span> <span class="n">Decimal</span><span class="p">(</span><span class="s1">'0.222222'</span><span class="p">))]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>which matches the expected result.</p>
|
||
</section>
|
||
<section id="coroutines-and-asynchronous-tasks">
|
||
<h3><a class="toc-backref" href="#coroutines-and-asynchronous-tasks" role="doc-backlink">Coroutines and Asynchronous Tasks</a></h3>
|
||
<p>Like generators, coroutines can yield and regain control. The major
|
||
difference from generators is that coroutines do not yield to the
|
||
immediate caller. Instead, the entire coroutine call stack
|
||
(coroutines chained by <code class="docutils literal notranslate"><span class="pre">await</span></code>) switches to another coroutine call
|
||
stack. In this regard, <code class="docutils literal notranslate"><span class="pre">await</span></code>-ing on a coroutine is conceptually
|
||
similar to a regular function call, and a coroutine chain
|
||
(or a “task”, e.g. an <code class="docutils literal notranslate"><span class="pre">asyncio.Task</span></code>) is conceptually similar to a
|
||
thread.</p>
|
||
<p>From this similarity we conclude that context variables in coroutines
|
||
should behave like “task locals”:</p>
|
||
<ul class="simple">
|
||
<li>changes to context variables in a coroutine are visible to the
|
||
coroutine that awaits on it;</li>
|
||
<li>changes to context variables made in the caller prior to awaiting
|
||
are visible to the awaited coroutine;</li>
|
||
<li>changes to context variables made in one task are not visible in
|
||
other tasks;</li>
|
||
<li>tasks spawned by other tasks inherit the execution context from the
|
||
parent task, but any changes to context variables made in the
|
||
parent task <em>after</em> the child task was spawned are <em>not</em> visible.</li>
|
||
</ul>
|
||
<p>The last point shows behaviour that is different from OS threads.
|
||
OS threads do not inherit the execution context by default.
|
||
There are two reasons for this: <em>common usage intent</em> and backwards
|
||
compatibility.</p>
|
||
<p>The main reason for why tasks inherit the context, and threads do
|
||
not, is the common usage intent. Tasks are often used for relatively
|
||
short-running operations which are logically tied to the code that
|
||
spawned the task (like running a coroutine with a timeout in
|
||
asyncio). OS threads, on the other hand, are normally used for
|
||
long-running, logically separate code.</p>
|
||
<p>With respect to backwards compatibility, we want the execution context
|
||
to behave like <code class="docutils literal notranslate"><span class="pre">threading.local()</span></code>. This is so that libraries can
|
||
start using the execution context in place of TLS with a lesser risk
|
||
of breaking compatibility with existing code.</p>
|
||
<p>Let’s review a few examples to illustrate the semantics we have just
|
||
defined.</p>
|
||
<p>Context variable propagation in a single task:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">asyncio</span>
|
||
|
||
<span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">async</span> <span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
<span class="k">await</span> <span class="n">sub</span><span class="p">()</span>
|
||
<span class="c1"># The effect of sub() is visible.</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'sub'</span>
|
||
|
||
<span class="k">async</span> <span class="k">def</span> <span class="nf">sub</span><span class="p">():</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main'</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'sub'</span><span class="p">)</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'sub'</span>
|
||
|
||
<span class="n">loop</span> <span class="o">=</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">get_event_loop</span><span class="p">()</span>
|
||
<span class="n">loop</span><span class="o">.</span><span class="n">run_until_complete</span><span class="p">(</span><span class="n">main</span><span class="p">())</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Context variable propagation between tasks:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">asyncio</span>
|
||
|
||
<span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">async</span> <span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
<span class="n">loop</span><span class="o">.</span><span class="n">create_task</span><span class="p">(</span><span class="n">sub</span><span class="p">())</span> <span class="c1"># schedules asynchronous execution</span>
|
||
<span class="c1"># of sub().</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main'</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main changed'</span><span class="p">)</span>
|
||
|
||
<span class="k">async</span> <span class="k">def</span> <span class="nf">sub</span><span class="p">():</span>
|
||
<span class="c1"># Sleeping will make sub() run after</span>
|
||
<span class="c1"># "var" is modified in main().</span>
|
||
<span class="k">await</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="c1"># The value of "var" is inherited from main(), but any</span>
|
||
<span class="c1"># changes to "var" made in main() after the task</span>
|
||
<span class="c1"># was created are *not* visible.</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main'</span>
|
||
|
||
<span class="c1"># This change is local to sub() and will not be visible</span>
|
||
<span class="c1"># to other tasks, including main().</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'sub'</span><span class="p">)</span>
|
||
|
||
<span class="n">loop</span> <span class="o">=</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">get_event_loop</span><span class="p">()</span>
|
||
<span class="n">loop</span><span class="o">.</span><span class="n">run_until_complete</span><span class="p">(</span><span class="n">main</span><span class="p">())</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>As shown above, changes to the execution context are local to the
|
||
task, and tasks get a snapshot of the execution context at the point
|
||
of creation.</p>
|
||
<p>There is one narrow edge case when this can lead to surprising
|
||
behaviour. Consider the following example where we modify the
|
||
context variable in a nested coroutine:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">async</span> <span class="k">def</span> <span class="nf">sub</span><span class="p">(</span><span class="n">var_value</span><span class="p">):</span>
|
||
<span class="k">await</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">var_value</span><span class="p">)</span>
|
||
|
||
<span class="k">async</span> <span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># waiting for sub() directly</span>
|
||
<span class="k">await</span> <span class="n">sub</span><span class="p">(</span><span class="s1">'sub-1'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># var change is visible</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'sub-1'</span>
|
||
|
||
<span class="c1"># waiting for sub() with a timeout;</span>
|
||
<span class="k">await</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">wait_for</span><span class="p">(</span><span class="n">sub</span><span class="p">(</span><span class="s1">'sub-2'</span><span class="p">),</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="c1"># wait_for() creates an implicit task, which isolates</span>
|
||
<span class="c1"># context changes, which means that the below assertion</span>
|
||
<span class="c1"># will fail.</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'sub-2'</span> <span class="c1"># AssertionError!</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>However, relying on context changes leaking to the caller is
|
||
ultimately a bad pattern. For this reason, the behaviour shown in
|
||
the above example is not considered a major issue and can be
|
||
addressed with proper documentation.</p>
|
||
</section>
|
||
</section>
|
||
<section id="detailed-specification">
|
||
<h2><a class="toc-backref" href="#detailed-specification" role="doc-backlink">Detailed Specification</a></h2>
|
||
<p>Conceptually, an <em>execution context</em> (EC) is a stack of logical
|
||
contexts. There is always exactly one active EC per Python thread.</p>
|
||
<p>A <em>logical context</em> (LC) is a mapping of context variables to their
|
||
values in that particular LC.</p>
|
||
<p>A <em>context variable</em> is an object representing a value in the
|
||
execution context. A new context variable object is created by
|
||
calling <code class="docutils literal notranslate"><span class="pre">contextvars.ContextVar(name:</span> <span class="pre">str)</span></code>. The value of the
|
||
required <code class="docutils literal notranslate"><span class="pre">name</span></code> argument is not used by the EC machinery, but may
|
||
be used for debugging and introspection.</p>
|
||
<p>The context variable object has the following methods and attributes:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">name</span></code>: the value passed to <code class="docutils literal notranslate"><span class="pre">ContextVar()</span></code>.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">get(*,</span> <span class="pre">topmost=False,</span> <span class="pre">default=None)</span></code>, if <em>topmost</em> is <code class="docutils literal notranslate"><span class="pre">False</span></code>
|
||
(the default), traverses the execution context top-to-bottom, until
|
||
the variable value is found. If <em>topmost</em> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, returns
|
||
the value of the variable in the topmost logical context.
|
||
If the variable value was not found, returns the value of <em>default</em>.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">set(value)</span></code>: sets the value of the variable in the topmost
|
||
logical context.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">delete()</span></code>: removes the variable from the topmost logical context.
|
||
Useful when restoring the logical context to the state prior to the
|
||
<code class="docutils literal notranslate"><span class="pre">set()</span></code> call, for example, in a context manager, see
|
||
<a class="reference internal" href="#setting-and-restoring-context-variables">Setting and restoring context variables</a> for more information.</li>
|
||
</ul>
|
||
<section id="id5">
|
||
<h3><a class="toc-backref" href="#id5" role="doc-backlink">Generators</a></h3>
|
||
<p>When created, each generator object has an empty logical context
|
||
object stored in its <code class="docutils literal notranslate"><span class="pre">__logical_context__</span></code> attribute. This logical
|
||
context is pushed onto the execution context at the beginning of each
|
||
generator iteration and popped at the end:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var1</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var1'</span><span class="p">)</span>
|
||
<span class="n">var2</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var2'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">gen</span><span class="p">():</span>
|
||
<span class="n">var1</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'var1-gen'</span><span class="p">)</span>
|
||
<span class="n">var2</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'var2-gen'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># EC = [</span>
|
||
<span class="c1"># outer_LC(),</span>
|
||
<span class="c1"># gen_LC({var1: 'var1-gen', var2: 'var2-gen'})</span>
|
||
<span class="c1"># ]</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">nested_gen</span><span class="p">()</span> <span class="c1"># nested_gen_LC is created</span>
|
||
<span class="nb">next</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="c1"># EC = [</span>
|
||
<span class="c1"># outer_LC(),</span>
|
||
<span class="c1"># gen_LC({var1: 'var1-gen', var2: 'var2-gen'})</span>
|
||
<span class="c1"># ]</span>
|
||
|
||
<span class="n">var1</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'var1-gen-mod'</span><span class="p">)</span>
|
||
<span class="n">var2</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'var2-gen-mod'</span><span class="p">)</span>
|
||
<span class="c1"># EC = [</span>
|
||
<span class="c1"># outer_LC(),</span>
|
||
<span class="c1"># gen_LC({var1: 'var1-gen-mod', var2: 'var2-gen-mod'})</span>
|
||
<span class="c1"># ]</span>
|
||
<span class="nb">next</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">nested_gen</span><span class="p">():</span>
|
||
<span class="c1"># EC = [</span>
|
||
<span class="c1"># outer_LC(),</span>
|
||
<span class="c1"># gen_LC({var1: 'var1-gen', var2: 'var2-gen'}),</span>
|
||
<span class="c1"># nested_gen_LC()</span>
|
||
<span class="c1"># ]</span>
|
||
<span class="k">assert</span> <span class="n">var1</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'var1-gen'</span>
|
||
<span class="k">assert</span> <span class="n">var2</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'var2-gen'</span>
|
||
|
||
<span class="n">var1</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'var1-nested-gen'</span><span class="p">)</span>
|
||
<span class="c1"># EC = [</span>
|
||
<span class="c1"># outer_LC(),</span>
|
||
<span class="c1"># gen_LC({var1: 'var1-gen', var2: 'var2-gen'}),</span>
|
||
<span class="c1"># nested_gen_LC({var1: 'var1-nested-gen'})</span>
|
||
<span class="c1"># ]</span>
|
||
<span class="k">yield</span>
|
||
|
||
<span class="c1"># EC = [</span>
|
||
<span class="c1"># outer_LC(),</span>
|
||
<span class="c1"># gen_LC({var1: 'var1-gen-mod', var2: 'var2-gen-mod'}),</span>
|
||
<span class="c1"># nested_gen_LC({var1: 'var1-nested-gen'})</span>
|
||
<span class="c1"># ]</span>
|
||
<span class="k">assert</span> <span class="n">var1</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'var1-nested-gen'</span>
|
||
<span class="k">assert</span> <span class="n">var2</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'var2-gen-mod'</span>
|
||
|
||
<span class="k">yield</span>
|
||
|
||
<span class="c1"># EC = [outer_LC()]</span>
|
||
|
||
<span class="n">g</span> <span class="o">=</span> <span class="n">gen</span><span class="p">()</span> <span class="c1"># gen_LC is created for the generator object `g`</span>
|
||
<span class="nb">list</span><span class="p">(</span><span class="n">g</span><span class="p">)</span>
|
||
|
||
<span class="c1"># EC = [outer_LC()]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The snippet above shows the state of the execution context stack
|
||
throughout the generator lifespan.</p>
|
||
</section>
|
||
<section id="contextlib-contextmanager">
|
||
<h3><a class="toc-backref" href="#contextlib-contextmanager" role="doc-backlink">contextlib.contextmanager</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">contextlib.contextmanager()</span></code> decorator can be used to turn
|
||
a generator into a context manager. A context manager that
|
||
temporarily modifies the value of a context variable could be defined
|
||
like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="nd">@contextlib</span><span class="o">.</span><span class="n">contextmanager</span>
|
||
<span class="k">def</span> <span class="nf">var_context</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
|
||
<span class="n">original_value</span> <span class="o">=</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
|
||
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||
<span class="k">yield</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">original_value</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Unfortunately, this would not work straight away, as the modification
|
||
to the <code class="docutils literal notranslate"><span class="pre">var</span></code> variable is contained to the <code class="docutils literal notranslate"><span class="pre">var_context()</span></code>
|
||
generator, and therefore will not be visible inside the <code class="docutils literal notranslate"><span class="pre">with</span></code>
|
||
block:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">func</span><span class="p">():</span>
|
||
<span class="c1"># EC = [{}, {}]</span>
|
||
|
||
<span class="k">with</span> <span class="n">var_context</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
|
||
<span class="c1"># EC becomes [{}, {}, {var: 10}] in the</span>
|
||
<span class="c1"># *precision_context()* generator,</span>
|
||
<span class="c1"># but here the EC is still [{}, {}]</span>
|
||
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="mi">10</span> <span class="c1"># AssertionError!</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The way to fix this is to set the generator’s <code class="docutils literal notranslate"><span class="pre">__logical_context__</span></code>
|
||
attribute to <code class="docutils literal notranslate"><span class="pre">None</span></code>. This will cause the generator to avoid
|
||
modifying the execution context stack.</p>
|
||
<p>We modify the <code class="docutils literal notranslate"><span class="pre">contextlib.contextmanager()</span></code> decorator to
|
||
set <code class="docutils literal notranslate"><span class="pre">genobj.__logical_context__</span></code> to <code class="docutils literal notranslate"><span class="pre">None</span></code> to produce
|
||
well-behaved context managers:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">func</span><span class="p">():</span>
|
||
<span class="c1"># EC = [{}, {}]</span>
|
||
|
||
<span class="k">with</span> <span class="n">var_context</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
|
||
<span class="c1"># EC = [{}, {var: 10}]</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="mi">10</span>
|
||
|
||
<span class="c1"># EC becomes [{}, {var: None}]</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="enumerating-context-vars">
|
||
<h3><a class="toc-backref" href="#enumerating-context-vars" role="doc-backlink">Enumerating context vars</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">ExecutionContext.vars()</span></code> method returns a list of
|
||
<code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> objects, that have values in the execution context.
|
||
This method is mostly useful for introspection and logging.</p>
|
||
</section>
|
||
<section id="coroutines">
|
||
<h3><a class="toc-backref" href="#coroutines" role="doc-backlink">coroutines</a></h3>
|
||
<p>In CPython, coroutines share the implementation with generators.
|
||
The difference is that in coroutines <code class="docutils literal notranslate"><span class="pre">__logical_context__</span></code> defaults
|
||
to <code class="docutils literal notranslate"><span class="pre">None</span></code>. This affects both the <code class="docutils literal notranslate"><span class="pre">async</span> <span class="pre">def</span></code> coroutines and the
|
||
old-style generator-based coroutines (generators decorated with
|
||
<code class="docutils literal notranslate"><span class="pre">@types.coroutine</span></code>).</p>
|
||
</section>
|
||
<section id="asynchronous-generators">
|
||
<h3><a class="toc-backref" href="#asynchronous-generators" role="doc-backlink">Asynchronous Generators</a></h3>
|
||
<p>The execution context semantics in asynchronous generators does not
|
||
differ from that of regular generators.</p>
|
||
</section>
|
||
<section id="asyncio">
|
||
<h3><a class="toc-backref" href="#asyncio" role="doc-backlink">asyncio</a></h3>
|
||
<p><code class="docutils literal notranslate"><span class="pre">asyncio</span></code> uses <code class="docutils literal notranslate"><span class="pre">Loop.call_soon</span></code>, <code class="docutils literal notranslate"><span class="pre">Loop.call_later</span></code>,
|
||
and <code class="docutils literal notranslate"><span class="pre">Loop.call_at</span></code> to schedule the asynchronous execution of a
|
||
function. <code class="docutils literal notranslate"><span class="pre">asyncio.Task</span></code> uses <code class="docutils literal notranslate"><span class="pre">call_soon()</span></code> to run the
|
||
wrapped coroutine.</p>
|
||
<p>We modify <code class="docutils literal notranslate"><span class="pre">Loop.call_{at,later,soon}</span></code> to accept the new
|
||
optional <em>execution_context</em> keyword argument, which defaults to
|
||
the copy of the current execution context:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">call_soon</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">callback</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">execution_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">execution_context</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">execution_context</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">get_execution_context</span><span class="p">()</span>
|
||
|
||
<span class="c1"># ... some time later</span>
|
||
|
||
<span class="n">contextvars</span><span class="o">.</span><span class="n">run_with_execution_context</span><span class="p">(</span>
|
||
<span class="n">execution_context</span><span class="p">,</span> <span class="n">callback</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">contextvars.get_execution_context()</span></code> function returns a
|
||
shallow copy of the current execution context. By shallow copy here
|
||
we mean such a new execution context that:</p>
|
||
<ul class="simple">
|
||
<li>lookups in the copy provide the same results as in the original
|
||
execution context, and</li>
|
||
<li>any changes in the original execution context do not affect the
|
||
copy, and</li>
|
||
<li>any changes to the copy do not affect the original execution
|
||
context.</li>
|
||
</ul>
|
||
<p>Either of the following satisfy the copy requirements:</p>
|
||
<ul class="simple">
|
||
<li>a new stack with shallow copies of logical contexts;</li>
|
||
<li>a new stack with one squashed logical context.</li>
|
||
</ul>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">contextvars.run_with_execution_context(ec,</span> <span class="pre">func,</span> <span class="pre">*args,</span>
|
||
<span class="pre">**kwargs)</span></code> function runs <code class="docutils literal notranslate"><span class="pre">func(*args,</span> <span class="pre">**kwargs)</span></code> with <em>ec</em> as the
|
||
execution context. The function performs the following steps:</p>
|
||
<ol class="arabic simple">
|
||
<li>Set <em>ec</em> as the current execution context stack in the current
|
||
thread.</li>
|
||
<li>Push an empty logical context onto the stack.</li>
|
||
<li>Run <code class="docutils literal notranslate"><span class="pre">func(*args,</span> <span class="pre">**kwargs)</span></code>.</li>
|
||
<li>Pop the logical context from the stack.</li>
|
||
<li>Restore the original execution context stack.</li>
|
||
<li>Return or raise the <code class="docutils literal notranslate"><span class="pre">func()</span></code> result.</li>
|
||
</ol>
|
||
<p>These steps ensure that <em>ec</em> cannot be modified by <em>func</em>,
|
||
which makes <code class="docutils literal notranslate"><span class="pre">run_with_execution_context()</span></code> idempotent.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">asyncio.Task</span></code> is modified as follows:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Task</span><span class="p">:</span>
|
||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coro</span><span class="p">):</span>
|
||
<span class="o">...</span>
|
||
<span class="c1"># Get the current execution context snapshot.</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">_exec_context</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">get_execution_context</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Create an empty Logical Context that will be</span>
|
||
<span class="c1"># used by coroutines run in the task.</span>
|
||
<span class="n">coro</span><span class="o">.</span><span class="n">__logical_context__</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">LogicalContext</span><span class="p">()</span>
|
||
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">_loop</span><span class="o">.</span><span class="n">call_soon</span><span class="p">(</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">_step</span><span class="p">,</span>
|
||
<span class="n">execution_context</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_exec_context</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
<span class="o">...</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">_loop</span><span class="o">.</span><span class="n">call_soon</span><span class="p">(</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">_step</span><span class="p">,</span>
|
||
<span class="n">execution_context</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_exec_context</span><span class="p">)</span>
|
||
<span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="generators-transformed-into-iterators">
|
||
<h3><a class="toc-backref" href="#generators-transformed-into-iterators" role="doc-backlink">Generators Transformed into Iterators</a></h3>
|
||
<p>Any Python generator can be represented as an equivalent iterator.
|
||
Compilers like Cython rely on this axiom. With respect to the
|
||
execution context, such iterator should behave the same way as the
|
||
generator it represents.</p>
|
||
<p>This means that there needs to be a Python API to create new logical
|
||
contexts and run code with a given logical context.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">contextvars.LogicalContext()</span></code> function creates a new empty
|
||
logical context.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">contextvars.run_with_logical_context(lc,</span> <span class="pre">func,</span> <span class="pre">*args,</span>
|
||
<span class="pre">**kwargs)</span></code> function can be used to run functions in the specified
|
||
logical context. The <em>lc</em> can be modified as a result of the call.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">contextvars.run_with_logical_context()</span></code> function performs the
|
||
following steps:</p>
|
||
<ol class="arabic simple">
|
||
<li>Push <em>lc</em> onto the current execution context stack.</li>
|
||
<li>Run <code class="docutils literal notranslate"><span class="pre">func(*args,</span> <span class="pre">**kwargs)</span></code>.</li>
|
||
<li>Pop <em>lc</em> from the execution context stack.</li>
|
||
<li>Return or raise the <code class="docutils literal notranslate"><span class="pre">func()</span></code> result.</li>
|
||
</ol>
|
||
<p>By using <code class="docutils literal notranslate"><span class="pre">LogicalContext()</span></code> and <code class="docutils literal notranslate"><span class="pre">run_with_logical_context()</span></code>,
|
||
we can replicate the generator behaviour like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Generator</span><span class="p">:</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">logical_context</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">LogicalContext</span><span class="p">()</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="bp">self</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__next__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">run_with_logical_context</span><span class="p">(</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">logical_context</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_next_impl</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">_next_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="c1"># Actual __next__ implementation.</span>
|
||
<span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Let’s see how this pattern can be applied to an example generator:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># create a new context variable</span>
|
||
<span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">gen_series</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">):</span>
|
||
<span class="k">yield</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">*</span> <span class="n">i</span>
|
||
|
||
<span class="c1"># gen_series is equivalent to the following iterator:</span>
|
||
|
||
<span class="k">class</span> <span class="nc">CompiledGenSeries</span><span class="p">:</span>
|
||
|
||
<span class="c1"># This class is what the `gen_series()` generator can</span>
|
||
<span class="c1"># be transformed to by a compiler like Cython.</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="p">):</span>
|
||
<span class="c1"># Create a new empty logical context,</span>
|
||
<span class="c1"># like the generators do.</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">logical_context</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">LogicalContext</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Initialize the generator in its LC.</span>
|
||
<span class="c1"># Otherwise `var.set(10)` in the `_init` method</span>
|
||
<span class="c1"># would leak.</span>
|
||
<span class="n">contextvars</span><span class="o">.</span><span class="n">run_with_logical_context</span><span class="p">(</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">logical_context</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_init</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">_init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">i</span> <span class="o">=</span> <span class="mi">1</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">n</span> <span class="o">=</span> <span class="n">n</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="bp">self</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__next__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="c1"># Run the actual implementation of __next__ in our LC.</span>
|
||
<span class="k">return</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">run_with_logical_context</span><span class="p">(</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">logical_context</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_next_impl</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">_next_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">i</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="ne">StopIteration</span>
|
||
|
||
<span class="n">result</span> <span class="o">=</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">i</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span>
|
||
<span class="k">return</span> <span class="n">result</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>For hand-written iterators such approach to context management is
|
||
normally not necessary, and it is easier to set and restore
|
||
context variables directly in <code class="docutils literal notranslate"><span class="pre">__next__</span></code>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MyIterator</span><span class="p">:</span>
|
||
|
||
<span class="c1"># ...</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__next__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="n">old_val</span> <span class="o">=</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">new_val</span><span class="p">)</span>
|
||
<span class="c1"># ...</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">old_val</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="implementation">
|
||
<h2><a class="toc-backref" href="#implementation" role="doc-backlink">Implementation</a></h2>
|
||
<p>Execution context is implemented as an immutable linked list of
|
||
logical contexts, where each logical context is an immutable weak key
|
||
mapping. A pointer to the currently active execution context is
|
||
stored in the OS thread state:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span> +-----------------+
|
||
| | ec
|
||
| PyThreadState +-------------+
|
||
| | |
|
||
+-----------------+ |
|
||
|
|
||
ec_node ec_node ec_node v
|
||
+------+------+ +------+------+ +------+------+
|
||
| NULL | lc |<----| prev | lc |<----| prev | lc |
|
||
+------+--+---+ +------+--+---+ +------+--+---+
|
||
| | |
|
||
LC v LC v LC v
|
||
+-------------+ +-------------+ +-------------+
|
||
| var1: obj1 | | EMPTY | | var1: obj4 |
|
||
| var2: obj2 | +-------------+ +-------------+
|
||
| var3: obj3 |
|
||
+-------------+
|
||
</pre></div>
|
||
</div>
|
||
<p>The choice of the immutable list of immutable mappings as a
|
||
fundamental data structure is motivated by the need to efficiently
|
||
implement <code class="docutils literal notranslate"><span class="pre">contextvars.get_execution_context()</span></code>, which is to be
|
||
frequently used by asynchronous tasks and callbacks. When the EC is
|
||
immutable, <code class="docutils literal notranslate"><span class="pre">get_execution_context()</span></code> can simply copy the current
|
||
execution context <em>by reference</em>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">get_execution_context</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">PyThreadState_Get</span><span class="p">()</span><span class="o">.</span><span class="n">ec</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Let’s review all possible context modification scenarios:</p>
|
||
<ul>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">ContextVariable.set()</span></code> method is called:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">ContextVar_set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">val</span><span class="p">):</span>
|
||
<span class="c1"># See a more complete set() definition</span>
|
||
<span class="c1"># in the `Context Variables` section.</span>
|
||
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
<span class="n">top_ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
<span class="n">top_lc</span> <span class="o">=</span> <span class="n">top_ec_node</span><span class="o">.</span><span class="n">lc</span>
|
||
<span class="n">new_top_lc</span> <span class="o">=</span> <span class="n">top_lc</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">ec_node</span><span class="p">(</span>
|
||
<span class="n">prev</span><span class="o">=</span><span class="n">top_ec_node</span><span class="o">.</span><span class="n">prev</span><span class="p">,</span>
|
||
<span class="n">lc</span><span class="o">=</span><span class="n">new_top_lc</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">contextvars.run_with_logical_context()</span></code> is called, in which
|
||
case the passed logical context object is appended to the execution
|
||
context:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">run_with_logical_context</span><span class="p">(</span><span class="n">lc</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
|
||
<span class="n">old_top_ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
<span class="n">new_top_ec_node</span> <span class="o">=</span> <span class="n">ec_node</span><span class="p">(</span><span class="n">prev</span><span class="o">=</span><span class="n">old_top_ec_node</span><span class="p">,</span> <span class="n">lc</span><span class="o">=</span><span class="n">lc</span><span class="p">)</span>
|
||
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">new_top_ec_node</span>
|
||
<span class="k">return</span> <span class="n">func</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">old_top_ec_node</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">contextvars.run_with_execution_context()</span></code> is called, in which
|
||
case the current execution context is set to the passed execution
|
||
context with a new empty logical context appended to it:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">run_with_execution_context</span><span class="p">(</span><span class="n">ec</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
|
||
<span class="n">old_top_ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
<span class="n">new_lc</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">LogicalContext</span><span class="p">()</span>
|
||
<span class="n">new_top_ec_node</span> <span class="o">=</span> <span class="n">ec_node</span><span class="p">(</span><span class="n">prev</span><span class="o">=</span><span class="n">ec</span><span class="p">,</span> <span class="n">lc</span><span class="o">=</span><span class="n">new_lc</span><span class="p">)</span>
|
||
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">new_top_ec_node</span>
|
||
<span class="k">return</span> <span class="n">func</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">old_top_ec_node</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li>Either <code class="docutils literal notranslate"><span class="pre">genobj.send()</span></code>, <code class="docutils literal notranslate"><span class="pre">genobj.throw()</span></code>, <code class="docutils literal notranslate"><span class="pre">genobj.close()</span></code>
|
||
are called on a <code class="docutils literal notranslate"><span class="pre">genobj</span></code> generator, in which case the logical
|
||
context recorded in <code class="docutils literal notranslate"><span class="pre">genobj</span></code> is pushed onto the stack:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">PyGen_New</span><span class="p">(</span><span class="n">PyGenObject</span> <span class="o">*</span><span class="n">gen</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="p">(</span><span class="n">gen</span><span class="o">.</span><span class="n">gi_code</span><span class="o">.</span><span class="n">co_flags</span> <span class="o">&</span>
|
||
<span class="p">(</span><span class="n">CO_COROUTINE</span> <span class="o">|</span> <span class="n">CO_ITERABLE_COROUTINE</span><span class="p">)):</span>
|
||
<span class="c1"># gen is an 'async def' coroutine, or a generator</span>
|
||
<span class="c1"># decorated with @types.coroutine.</span>
|
||
<span class="n">gen</span><span class="o">.</span><span class="n">__logical_context__</span> <span class="o">=</span> <span class="kc">None</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># Non-coroutine generator</span>
|
||
<span class="n">gen</span><span class="o">.</span><span class="n">__logical_context__</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">LogicalContext</span><span class="p">()</span>
|
||
|
||
<span class="n">gen_send</span><span class="p">(</span><span class="n">PyGenObject</span> <span class="o">*</span><span class="n">gen</span><span class="p">,</span> <span class="o">...</span><span class="p">):</span>
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
|
||
<span class="k">if</span> <span class="n">gen</span><span class="o">.</span><span class="n">__logical_context__</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">old_top_ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
<span class="n">new_top_ec_node</span> <span class="o">=</span> <span class="n">ec_node</span><span class="p">(</span>
|
||
<span class="n">prev</span><span class="o">=</span><span class="n">old_top_ec_node</span><span class="p">,</span>
|
||
<span class="n">lc</span><span class="o">=</span><span class="n">gen</span><span class="o">.</span><span class="n">__logical_context__</span><span class="p">)</span>
|
||
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">new_top_ec_node</span>
|
||
<span class="k">return</span> <span class="n">_gen_send_impl</span><span class="p">(</span><span class="n">gen</span><span class="p">,</span> <span class="o">...</span><span class="p">)</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="n">gen</span><span class="o">.</span><span class="n">__logical_context__</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span><span class="o">.</span><span class="n">lc</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">old_top_ec_node</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">_gen_send_impl</span><span class="p">(</span><span class="n">gen</span><span class="p">,</span> <span class="o">...</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li>Coroutines and asynchronous generators share the implementation
|
||
with generators, and the above changes apply to them as well.</li>
|
||
</ul>
|
||
<p>In certain scenarios the EC may need to be squashed to limit the
|
||
size of the chain. For example, consider the following corner case:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">async</span> <span class="k">def</span> <span class="nf">repeat</span><span class="p">(</span><span class="n">coro</span><span class="p">,</span> <span class="n">delay</span><span class="p">):</span>
|
||
<span class="k">await</span> <span class="n">coro</span><span class="p">()</span>
|
||
<span class="k">await</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">delay</span><span class="p">)</span>
|
||
<span class="n">loop</span><span class="o">.</span><span class="n">create_task</span><span class="p">(</span><span class="n">repeat</span><span class="p">(</span><span class="n">coro</span><span class="p">,</span> <span class="n">delay</span><span class="p">))</span>
|
||
|
||
<span class="k">async</span> <span class="k">def</span> <span class="nf">ping</span><span class="p">():</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'ping'</span><span class="p">)</span>
|
||
|
||
<span class="n">loop</span> <span class="o">=</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">get_event_loop</span><span class="p">()</span>
|
||
<span class="n">loop</span><span class="o">.</span><span class="n">create_task</span><span class="p">(</span><span class="n">repeat</span><span class="p">(</span><span class="n">ping</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">loop</span><span class="o">.</span><span class="n">run_forever</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>In the above code, the EC chain will grow as long as <code class="docutils literal notranslate"><span class="pre">repeat()</span></code> is
|
||
called. Each new task will call
|
||
<code class="docutils literal notranslate"><span class="pre">contextvars.run_with_execution_context()</span></code>, which will append a new
|
||
logical context to the chain. To prevent unbounded growth,
|
||
<code class="docutils literal notranslate"><span class="pre">contextvars.get_execution_context()</span></code> checks if the chain
|
||
is longer than a predetermined maximum, and if it is, squashes the
|
||
chain into a single LC:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">get_execution_context</span><span class="p">():</span>
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
|
||
<span class="k">if</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec_len</span> <span class="o">></span> <span class="n">EC_LEN_MAX</span><span class="p">:</span>
|
||
<span class="n">squashed_lc</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">LogicalContext</span><span class="p">()</span>
|
||
|
||
<span class="n">ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
<span class="k">while</span> <span class="n">ec_node</span><span class="p">:</span>
|
||
<span class="c1"># The LC.merge() method does not replace</span>
|
||
<span class="c1"># existing keys.</span>
|
||
<span class="n">squashed_lc</span> <span class="o">=</span> <span class="n">squashed_lc</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">ec_node</span><span class="o">.</span><span class="n">lc</span><span class="p">)</span>
|
||
<span class="n">ec_node</span> <span class="o">=</span> <span class="n">ec_node</span><span class="o">.</span><span class="n">prev</span>
|
||
|
||
<span class="k">return</span> <span class="n">ec_node</span><span class="p">(</span><span class="n">prev</span><span class="o">=</span><span class="n">NULL</span><span class="p">,</span> <span class="n">lc</span><span class="o">=</span><span class="n">squashed_lc</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
</pre></div>
|
||
</div>
|
||
<section id="logical-context">
|
||
<h3><a class="toc-backref" href="#logical-context" role="doc-backlink">Logical Context</a></h3>
|
||
<p>Logical context is an immutable weak key mapping which has the
|
||
following properties with respect to garbage collection:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> objects are strongly-referenced only from the
|
||
application code, not from any of the execution context machinery
|
||
or values they point to. This means that there are no reference
|
||
cycles that could extend their lifespan longer than necessary, or
|
||
prevent their collection by the GC.</li>
|
||
<li>Values put in the execution context are guaranteed to be kept
|
||
alive while there is a <code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> key referencing them in
|
||
the thread.</li>
|
||
<li>If a <code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> is garbage collected, all of its values will
|
||
be removed from all contexts, allowing them to be GCed if needed.</li>
|
||
<li>If an OS thread has ended its execution, its thread state will be
|
||
cleaned up along with its execution context, cleaning
|
||
up all values bound to all context variables in the thread.</li>
|
||
</ul>
|
||
<p>As discussed earlier, we need <code class="docutils literal notranslate"><span class="pre">contextvars.get_execution_context()</span></code>
|
||
to be consistently fast regardless of the size of the execution
|
||
context, so logical context is necessarily an immutable mapping.</p>
|
||
<p>Choosing <code class="docutils literal notranslate"><span class="pre">dict</span></code> for the underlying implementation is suboptimal,
|
||
because <code class="docutils literal notranslate"><span class="pre">LC.set()</span></code> will cause <code class="docutils literal notranslate"><span class="pre">dict.copy()</span></code>, which is an O(N)
|
||
operation, where <em>N</em> is the number of items in the LC.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">get_execution_context()</span></code>, when squashing the EC, is an O(M)
|
||
operation, where <em>M</em> is the total number of context variable values
|
||
in the EC.</p>
|
||
<p>So, instead of <code class="docutils literal notranslate"><span class="pre">dict</span></code>, we choose Hash Array Mapped Trie (HAMT)
|
||
as the underlying implementation of logical contexts. (Scala and
|
||
Clojure use HAMT to implement high performance immutable collections
|
||
<a class="footnote-reference brackets" href="#id32" id="id6">[5]</a>, <a class="footnote-reference brackets" href="#id33" id="id7">[6]</a>.)</p>
|
||
<p>With HAMT <code class="docutils literal notranslate"><span class="pre">.set()</span></code> becomes an O(log N) operation, and
|
||
<code class="docutils literal notranslate"><span class="pre">get_execution_context()</span></code> squashing is more efficient on average due
|
||
to structural sharing in HAMT.</p>
|
||
<p>See <a class="reference internal" href="#appendix-hamt-performance-analysis">Appendix: HAMT Performance Analysis</a> for a more elaborate
|
||
analysis of HAMT performance compared to <code class="docutils literal notranslate"><span class="pre">dict</span></code>.</p>
|
||
</section>
|
||
<section id="context-variables">
|
||
<h3><a class="toc-backref" href="#context-variables" role="doc-backlink">Context Variables</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">ContextVar.get()</span></code> and <code class="docutils literal notranslate"><span class="pre">ContextVar.set()</span></code> methods are
|
||
implemented as follows (in pseudo-code):</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">ContextVar</span><span class="p">:</span>
|
||
|
||
<span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">topmost</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
|
||
<span class="n">ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
<span class="k">while</span> <span class="n">ec_node</span><span class="p">:</span>
|
||
<span class="k">if</span> <span class="bp">self</span> <span class="ow">in</span> <span class="n">ec_node</span><span class="o">.</span><span class="n">lc</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">ec_node</span><span class="o">.</span><span class="n">lc</span><span class="p">[</span><span class="bp">self</span><span class="p">]</span>
|
||
<span class="k">if</span> <span class="n">topmost</span><span class="p">:</span>
|
||
<span class="k">break</span>
|
||
<span class="n">ec_node</span> <span class="o">=</span> <span class="n">ec_node</span><span class="o">.</span><span class="n">prev</span>
|
||
|
||
<span class="k">return</span> <span class="n">default</span>
|
||
|
||
<span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
<span class="n">top_ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
|
||
<span class="k">if</span> <span class="n">top_ec_node</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">top_lc</span> <span class="o">=</span> <span class="n">top_ec_node</span><span class="o">.</span><span class="n">lc</span>
|
||
<span class="n">new_top_lc</span> <span class="o">=</span> <span class="n">top_lc</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">ec_node</span><span class="p">(</span>
|
||
<span class="n">prev</span><span class="o">=</span><span class="n">top_ec_node</span><span class="o">.</span><span class="n">prev</span><span class="p">,</span>
|
||
<span class="n">lc</span><span class="o">=</span><span class="n">new_top_lc</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># First ContextVar.set() in this OS thread.</span>
|
||
<span class="n">top_lc</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">LogicalContext</span><span class="p">()</span>
|
||
<span class="n">new_top_lc</span> <span class="o">=</span> <span class="n">top_lc</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">ec_node</span><span class="p">(</span>
|
||
<span class="n">prev</span><span class="o">=</span><span class="n">NULL</span><span class="p">,</span>
|
||
<span class="n">lc</span><span class="o">=</span><span class="n">new_top_lc</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
<span class="n">top_ec_node</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">ec</span>
|
||
|
||
<span class="k">if</span> <span class="n">top_ec_node</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="ne">LookupError</span>
|
||
|
||
<span class="n">top_lc</span> <span class="o">=</span> <span class="n">top_ec_node</span><span class="o">.</span><span class="n">lc</span>
|
||
<span class="k">if</span> <span class="bp">self</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">top_lc</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="ne">LookupError</span>
|
||
|
||
<span class="n">new_top_lc</span> <span class="o">=</span> <span class="n">top_lc</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||
|
||
<span class="n">tstate</span><span class="o">.</span><span class="n">ec</span> <span class="o">=</span> <span class="n">ec_node</span><span class="p">(</span>
|
||
<span class="n">prev</span><span class="o">=</span><span class="n">top_ec_node</span><span class="o">.</span><span class="n">prev</span><span class="p">,</span>
|
||
<span class="n">lc</span><span class="o">=</span><span class="n">new_top_lc</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>For efficient access in performance-sensitive code paths, such as in
|
||
<code class="docutils literal notranslate"><span class="pre">numpy</span></code> and <code class="docutils literal notranslate"><span class="pre">decimal</span></code>, we cache lookups in <code class="docutils literal notranslate"><span class="pre">ContextVar.get()</span></code>,
|
||
making it an O(1) operation when the cache is hit. The cache key is
|
||
composed from the following:</p>
|
||
<ul class="simple">
|
||
<li>The new <code class="docutils literal notranslate"><span class="pre">uint64_t</span> <span class="pre">PyThreadState->unique_id</span></code>, which is a globally
|
||
unique thread state identifier. It is computed from the new
|
||
<code class="docutils literal notranslate"><span class="pre">uint64_t</span> <span class="pre">PyInterpreterState->ts_counter</span></code>, which is incremented
|
||
whenever a new thread state is created.</li>
|
||
<li>The new <code class="docutils literal notranslate"><span class="pre">uint64_t</span> <span class="pre">PyThreadState->stack_version</span></code>, which is a
|
||
thread-specific counter, which is incremented whenever a non-empty
|
||
logical context is pushed onto the stack or popped from the stack.</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">uint64_t</span> <span class="pre">ContextVar->version</span></code> counter, which is incremented
|
||
whenever the context variable value is changed in any logical
|
||
context in any OS thread.</li>
|
||
</ul>
|
||
<p>The cache is then implemented as follows:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">ContextVar</span><span class="p">:</span>
|
||
|
||
<span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
|
||
<span class="o">...</span> <span class="c1"># implementation</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">version</span> <span class="o">+=</span> <span class="mi">1</span>
|
||
|
||
<span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">topmost</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">topmost</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_uncached</span><span class="p">(</span>
|
||
<span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">,</span> <span class="n">topmost</span><span class="o">=</span><span class="n">topmost</span><span class="p">)</span>
|
||
|
||
<span class="n">tstate</span> <span class="o">=</span> <span class="n">PyThreadState_Get</span><span class="p">()</span>
|
||
<span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_tstate_id</span> <span class="o">==</span> <span class="n">tstate</span><span class="o">.</span><span class="n">unique_id</span> <span class="ow">and</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">last_stack_ver</span> <span class="o">==</span> <span class="n">tstate</span><span class="o">.</span><span class="n">stack_version</span> <span class="ow">and</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">last_version</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">version</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_value</span>
|
||
|
||
<span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_uncached</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">)</span>
|
||
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">last_value</span> <span class="o">=</span> <span class="n">value</span> <span class="c1"># borrowed ref</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">last_tstate_id</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">unique_id</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">last_stack_version</span> <span class="o">=</span> <span class="n">tstate</span><span class="o">.</span><span class="n">stack_version</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">last_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">version</span>
|
||
|
||
<span class="k">return</span> <span class="n">value</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Note that <code class="docutils literal notranslate"><span class="pre">last_value</span></code> is a borrowed reference. We assume that
|
||
if the version checks are fine, the value object will be alive.
|
||
This allows the values of context variables to be properly garbage
|
||
collected.</p>
|
||
<p>This generic caching approach is similar to what the current C
|
||
implementation of <code class="docutils literal notranslate"><span class="pre">decimal</span></code> does to cache the current decimal
|
||
context, and has similar performance characteristics.</p>
|
||
</section>
|
||
</section>
|
||
<section id="performance-considerations">
|
||
<h2><a class="toc-backref" href="#performance-considerations" role="doc-backlink">Performance Considerations</a></h2>
|
||
<p>Tests of the reference implementation based on the prior
|
||
revisions of this PEP have shown 1-2% slowdown on generator
|
||
microbenchmarks and no noticeable difference in macrobenchmarks.</p>
|
||
<p>The performance of non-generator and non-async code is not
|
||
affected by this PEP.</p>
|
||
</section>
|
||
<section id="summary-of-the-new-apis">
|
||
<h2><a class="toc-backref" href="#summary-of-the-new-apis" role="doc-backlink">Summary of the New APIs</a></h2>
|
||
<section id="python">
|
||
<h3><a class="toc-backref" href="#python" role="doc-backlink">Python</a></h3>
|
||
<p>The following new Python APIs are introduced by this PEP:</p>
|
||
<ol class="arabic simple">
|
||
<li>The new <code class="docutils literal notranslate"><span class="pre">contextvars.ContextVar(name:</span> <span class="pre">str='...')</span></code> class,
|
||
instances of which have the following:<ul class="simple">
|
||
<li>the read-only <code class="docutils literal notranslate"><span class="pre">.name</span></code> attribute,</li>
|
||
<li>the <code class="docutils literal notranslate"><span class="pre">.get()</span></code> method, which returns the value of the variable
|
||
in the current execution context;</li>
|
||
<li>the <code class="docutils literal notranslate"><span class="pre">.set()</span></code> method, which sets the value of the variable in
|
||
the current logical context;</li>
|
||
<li>the <code class="docutils literal notranslate"><span class="pre">.delete()</span></code> method, which removes the value of the variable
|
||
from the current logical context.</li>
|
||
</ul>
|
||
</li>
|
||
<li>The new <code class="docutils literal notranslate"><span class="pre">contextvars.ExecutionContext()</span></code> class, which represents
|
||
an execution context.</li>
|
||
<li>The new <code class="docutils literal notranslate"><span class="pre">contextvars.LogicalContext()</span></code> class, which represents
|
||
a logical context.</li>
|
||
<li>The new <code class="docutils literal notranslate"><span class="pre">contextvars.get_execution_context()</span></code> function, which
|
||
returns an <code class="docutils literal notranslate"><span class="pre">ExecutionContext</span></code> instance representing a copy of
|
||
the current execution context.</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">contextvars.run_with_execution_context(ec:</span> <span class="pre">ExecutionContext,</span>
|
||
<span class="pre">func,</span> <span class="pre">*args,</span> <span class="pre">**kwargs)</span></code> function, which runs <em>func</em> with the
|
||
provided execution context.</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">contextvars.run_with_logical_context(lc:</span> <span class="pre">LogicalContext,</span>
|
||
<span class="pre">func,</span> <span class="pre">*args,</span> <span class="pre">**kwargs)</span></code> function, which runs <em>func</em> with the
|
||
provided logical context on top of the current execution context.</li>
|
||
</ol>
|
||
</section>
|
||
<section id="c-api">
|
||
<h3><a class="toc-backref" href="#c-api" role="doc-backlink">C API</a></h3>
|
||
<ol class="arabic simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyContextVar</span> <span class="pre">*</span> <span class="pre">PyContext_NewVar(char</span> <span class="pre">*desc)</span></code>: create a
|
||
<code class="docutils literal notranslate"><span class="pre">PyContextVar</span></code> object.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject</span> <span class="pre">*</span> <span class="pre">PyContext_GetValue(PyContextVar</span> <span class="pre">*,</span> <span class="pre">int</span> <span class="pre">topmost)</span></code>:
|
||
return the value of the variable in the current execution context.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">int</span> <span class="pre">PyContext_SetValue(PyContextVar</span> <span class="pre">*,</span> <span class="pre">PyObject</span> <span class="pre">*)</span></code>: set
|
||
the value of the variable in the current logical context.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">int</span> <span class="pre">PyContext_DelValue(PyContextVar</span> <span class="pre">*)</span></code>: delete the value of
|
||
the variable from the current logical context.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyLogicalContext</span> <span class="pre">*</span> <span class="pre">PyLogicalContext_New()</span></code>: create a new empty
|
||
<code class="docutils literal notranslate"><span class="pre">PyLogicalContext</span></code>.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyExecutionContext</span> <span class="pre">*</span> <span class="pre">PyExecutionContext_New()</span></code>: create a new
|
||
empty <code class="docutils literal notranslate"><span class="pre">PyExecutionContext</span></code>.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyExecutionContext</span> <span class="pre">*</span> <span class="pre">PyExecutionContext_Get()</span></code>: return the
|
||
current execution context.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">int</span> <span class="pre">PyContext_SetCurrent(</span>
|
||
<span class="pre">PyExecutionContext</span> <span class="pre">*,</span> <span class="pre">PyLogicalContext</span> <span class="pre">*)</span></code>: set the
|
||
passed EC object as the current execution context for the active
|
||
thread state, and/or set the passed LC object as the current
|
||
logical context.</li>
|
||
</ol>
|
||
</section>
|
||
</section>
|
||
<section id="design-considerations">
|
||
<h2><a class="toc-backref" href="#design-considerations" role="doc-backlink">Design Considerations</a></h2>
|
||
<section id="should-yield-from-leak-context-changes">
|
||
<h3><a class="toc-backref" href="#should-yield-from-leak-context-changes" role="doc-backlink">Should “yield from” leak context changes?</a></h3>
|
||
<p>No. It may be argued that <code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code> is semantically
|
||
equivalent to calling a function, and should leak context changes.
|
||
However, it is not possible to satisfy the following at the same time:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">next(gen)</span></code> <em>does not</em> leak context changes made in <code class="docutils literal notranslate"><span class="pre">gen</span></code>, and</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span> <span class="pre">gen</span></code> <em>leaks</em> context changes made in <code class="docutils literal notranslate"><span class="pre">gen</span></code>.</li>
|
||
</ul>
|
||
<p>The reason is that <code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code> can be used with a partially
|
||
iterated generator, which already has local context changes:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">gen</span><span class="p">():</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'gen'</span><span class="p">)</span>
|
||
<span class="k">yield</span> <span class="n">i</span>
|
||
|
||
<span class="k">def</span> <span class="nf">outer_gen</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'outer_gen'</span><span class="p">)</span>
|
||
<span class="n">g</span> <span class="o">=</span> <span class="n">gen</span><span class="p">()</span>
|
||
|
||
<span class="k">yield</span> <span class="nb">next</span><span class="p">(</span><span class="n">g</span><span class="p">)</span>
|
||
<span class="c1"># Changes not visible during partial iteration,</span>
|
||
<span class="c1"># the goal of this PEP:</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'outer_gen'</span>
|
||
|
||
<span class="k">yield from</span> <span class="n">g</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'outer_gen'</span> <span class="c1"># or 'gen'?</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Another example would be refactoring of an explicit <code class="docutils literal notranslate"><span class="pre">for..in</span> <span class="pre">yield</span></code>
|
||
construct to a <code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code> expression. Consider the following
|
||
code:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">outer_gen</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'outer_gen'</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">gen</span><span class="p">():</span>
|
||
<span class="k">yield</span> <span class="n">i</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'outer_gen'</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>which we want to refactor to use <code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">outer_gen</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'outer_gen'</span><span class="p">)</span>
|
||
|
||
<span class="k">yield from</span> <span class="n">gen</span><span class="p">()</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'outer_gen'</span> <span class="c1"># or 'gen'?</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The above examples illustrate that it is unsafe to refactor
|
||
generator code using <code class="docutils literal notranslate"><span class="pre">yield</span> <span class="pre">from</span></code> when it can leak context changes.</p>
|
||
<p>Thus, the only well-defined and consistent behaviour is to
|
||
<strong>always</strong> isolate context changes in generators, regardless of
|
||
how they are being iterated.</p>
|
||
</section>
|
||
<section id="should-pythreadstate-getdict-use-the-execution-context">
|
||
<h3><a class="toc-backref" href="#should-pythreadstate-getdict-use-the-execution-context" role="doc-backlink">Should <code class="docutils literal notranslate"><span class="pre">PyThreadState_GetDict()</span></code> use the execution context?</a></h3>
|
||
<p>No. <code class="docutils literal notranslate"><span class="pre">PyThreadState_GetDict</span></code> is based on TLS, and changing its
|
||
semantics will break backwards compatibility.</p>
|
||
</section>
|
||
<section id="pep-521">
|
||
<h3><a class="toc-backref" href="#pep-521" role="doc-backlink">PEP 521</a></h3>
|
||
<p><a class="pep reference internal" href="../pep-0521/" title="PEP 521 – Managing global context via ‘with’ blocks in generators and coroutines">PEP 521</a> proposes an alternative solution to the problem, which
|
||
extends the context manager protocol with two new methods:
|
||
<code class="docutils literal notranslate"><span class="pre">__suspend__()</span></code> and <code class="docutils literal notranslate"><span class="pre">__resume__()</span></code>. Similarly, the asynchronous
|
||
context manager protocol is also extended with <code class="docutils literal notranslate"><span class="pre">__asuspend__()</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">__aresume__()</span></code>.</p>
|
||
<p>This allows implementing context managers that manage non-local state,
|
||
which behave correctly in generators and coroutines.</p>
|
||
<p>For example, consider the following context manager, which uses
|
||
execution state:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Context</span><span class="p">:</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">old_x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'something'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">err</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">old_x</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>An equivalent implementation with <a class="pep reference internal" href="../pep-0521/" title="PEP 521 – Managing global context via ‘with’ blocks in generators and coroutines">PEP 521</a>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">local</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">local</span><span class="p">()</span>
|
||
|
||
<span class="k">class</span> <span class="nc">Context</span><span class="p">:</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">old_x</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">local</span><span class="p">,</span> <span class="s1">'x'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||
<span class="n">local</span><span class="o">.</span><span class="n">x</span> <span class="o">=</span> <span class="s1">'something'</span>
|
||
|
||
<span class="k">def</span> <span class="nf">__suspend__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="n">local</span><span class="o">.</span><span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">old_x</span>
|
||
|
||
<span class="k">def</span> <span class="nf">__resume__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="n">local</span><span class="o">.</span><span class="n">x</span> <span class="o">=</span> <span class="s1">'something'</span>
|
||
|
||
<span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">err</span><span class="p">):</span>
|
||
<span class="n">local</span><span class="o">.</span><span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">old_x</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The downside of this approach is the addition of significant new
|
||
complexity to the context manager protocol and the interpreter
|
||
implementation. This approach is also likely to negatively impact
|
||
the performance of generators and coroutines.</p>
|
||
<p>Additionally, the solution in <a class="pep reference internal" href="../pep-0521/" title="PEP 521 – Managing global context via ‘with’ blocks in generators and coroutines">PEP 521</a> is limited to context
|
||
managers, and does not provide any mechanism to propagate state in
|
||
asynchronous tasks and callbacks.</p>
|
||
</section>
|
||
<section id="can-execution-context-be-implemented-without-modifying-cpython">
|
||
<h3><a class="toc-backref" href="#can-execution-context-be-implemented-without-modifying-cpython" role="doc-backlink">Can Execution Context be implemented without modifying CPython?</a></h3>
|
||
<p>No.</p>
|
||
<p>It is true that the concept of “task-locals” can be implemented
|
||
for coroutines in libraries (see, for example, <a class="footnote-reference brackets" href="#id50" id="id8">[29]</a> and <a class="footnote-reference brackets" href="#id51" id="id9">[30]</a>).
|
||
On the other hand, generators are managed by the Python interpreter
|
||
directly, and so their context must also be managed by the
|
||
interpreter.</p>
|
||
<p>Furthermore, execution context cannot be implemented in a third-party
|
||
module at all, otherwise the standard library, including <code class="docutils literal notranslate"><span class="pre">decimal</span></code>
|
||
would not be able to rely on it.</p>
|
||
</section>
|
||
<section id="should-we-update-sys-displayhook-and-other-apis-to-use-ec">
|
||
<h3><a class="toc-backref" href="#should-we-update-sys-displayhook-and-other-apis-to-use-ec" role="doc-backlink">Should we update sys.displayhook and other APIs to use EC?</a></h3>
|
||
<p>APIs like redirecting stdout by overwriting <code class="docutils literal notranslate"><span class="pre">sys.stdout</span></code>, or
|
||
specifying new exception display hooks by overwriting the
|
||
<code class="docutils literal notranslate"><span class="pre">sys.displayhook</span></code> function are affecting the whole Python process
|
||
<strong>by design</strong>. Their users assume that the effect of changing
|
||
them will be visible across OS threads. Therefore, we cannot
|
||
just make these APIs to use the new Execution Context.</p>
|
||
<p>That said we think it is possible to design new APIs that will
|
||
be context aware, but that is outside of the scope of this PEP.</p>
|
||
</section>
|
||
<section id="greenlets">
|
||
<h3><a class="toc-backref" href="#greenlets" role="doc-backlink">Greenlets</a></h3>
|
||
<p>Greenlet is an alternative implementation of cooperative
|
||
scheduling for Python. Although greenlet package is not part of
|
||
CPython, popular frameworks like gevent rely on it, and it is
|
||
important that greenlet can be modified to support execution
|
||
contexts.</p>
|
||
<p>Conceptually, the behaviour of greenlets is very similar to that of
|
||
generators, which means that similar changes around greenlet entry
|
||
and exit can be done to add support for execution context. This
|
||
PEP provides the necessary C APIs to do that.</p>
|
||
</section>
|
||
<section id="context-manager-as-the-interface-for-modifications">
|
||
<h3><a class="toc-backref" href="#context-manager-as-the-interface-for-modifications" role="doc-backlink">Context manager as the interface for modifications</a></h3>
|
||
<p>This PEP concentrates on the low-level mechanics and the minimal
|
||
API that enables fundamental operations with execution context.</p>
|
||
<p>For developer convenience, a high-level context manager interface
|
||
may be added to the <code class="docutils literal notranslate"><span class="pre">contextvars</span></code> module. For example:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">with</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">set_var</span><span class="p">(</span><span class="n">var</span><span class="p">,</span> <span class="s1">'foo'</span><span class="p">):</span>
|
||
<span class="c1"># ...</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="setting-and-restoring-context-variables">
|
||
<h3><a class="toc-backref" href="#setting-and-restoring-context-variables" role="doc-backlink">Setting and restoring context variables</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">ContextVar.delete()</span></code> method removes the context variable from
|
||
the topmost logical context.</p>
|
||
<p>If the variable is not found in the topmost logical context, a
|
||
<code class="docutils literal notranslate"><span class="pre">LookupError</span></code> is raised, similarly to <code class="docutils literal notranslate"><span class="pre">del</span> <span class="pre">var</span></code> raising
|
||
<code class="docutils literal notranslate"><span class="pre">NameError</span></code> when <code class="docutils literal notranslate"><span class="pre">var</span></code> is not in scope.</p>
|
||
<p>This method is useful when there is a (rare) need to correctly restore
|
||
the state of a logical context, such as when a nested generator
|
||
wants to modify the logical context <em>temporarily</em>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">gen</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">some_var_context_manager</span><span class="p">(</span><span class="s1">'gen'</span><span class="p">):</span>
|
||
<span class="c1"># EC = [{var: 'main'}, {var: 'gen'}]</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'gen'</span>
|
||
<span class="k">yield</span>
|
||
|
||
<span class="c1"># EC = [{var: 'main modified'}, {}]</span>
|
||
<span class="k">assert</span> <span class="n">var</span><span class="o">.</span><span class="n">get</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'main modified'</span>
|
||
<span class="k">yield</span>
|
||
|
||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main'</span><span class="p">)</span>
|
||
<span class="n">g</span> <span class="o">=</span> <span class="n">gen</span><span class="p">()</span>
|
||
<span class="nb">next</span><span class="p">(</span><span class="n">g</span><span class="p">)</span>
|
||
<span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'main modified'</span><span class="p">)</span>
|
||
<span class="nb">next</span><span class="p">(</span><span class="n">g</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The above example would work correctly only if there is a way to
|
||
delete <code class="docutils literal notranslate"><span class="pre">var</span></code> from the logical context in <code class="docutils literal notranslate"><span class="pre">gen()</span></code>. Setting it
|
||
to a “previous value” in <code class="docutils literal notranslate"><span class="pre">__exit__()</span></code> would mask changes made
|
||
in <code class="docutils literal notranslate"><span class="pre">main()</span></code> between the iterations.</p>
|
||
</section>
|
||
<section id="alternative-designs-for-contextvar-api">
|
||
<h3><a class="toc-backref" href="#alternative-designs-for-contextvar-api" role="doc-backlink">Alternative Designs for ContextVar API</a></h3>
|
||
<section id="logical-context-with-stacked-values">
|
||
<h4><a class="toc-backref" href="#logical-context-with-stacked-values" role="doc-backlink">Logical Context with stacked values</a></h4>
|
||
<p>By the design presented in this PEP, logical context is a simple
|
||
<code class="docutils literal notranslate"><span class="pre">LC({ContextVar:</span> <span class="pre">value,</span> <span class="pre">...})</span></code> mapping. An alternative
|
||
representation is to store a stack of values for each context
|
||
variable: <code class="docutils literal notranslate"><span class="pre">LC({ContextVar:</span> <span class="pre">[val1,</span> <span class="pre">val2,</span> <span class="pre">...],</span> <span class="pre">...})</span></code>.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> methods would then be:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">get(*,</span> <span class="pre">default=None)</span></code> – traverses the stack
|
||
of logical contexts, and returns the top value from the
|
||
first non-empty logical context;</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">push(val)</span></code> – pushes <em>val</em> onto the stack of values in the
|
||
current logical context;</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">pop()</span></code> – pops the top value from the stack of values in
|
||
the current logical context.</li>
|
||
</ul>
|
||
<p>Compared to the single-value design with the <code class="docutils literal notranslate"><span class="pre">set()</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">delete()</span></code> methods, the stack-based approach allows for a simpler
|
||
implementation of the set/restore pattern. However, the mental
|
||
burden of this approach is considered to be higher, since there
|
||
would be <em>two</em> stacks to consider: a stack of LCs and a stack of
|
||
values in each LC.</p>
|
||
<p>(This idea was suggested by Nathaniel Smith.)</p>
|
||
</section>
|
||
<section id="contextvar-set-reset">
|
||
<h4><a class="toc-backref" href="#contextvar-set-reset" role="doc-backlink">ContextVar “set/reset”</a></h4>
|
||
<p>Yet another approach is to return a special object from
|
||
<code class="docutils literal notranslate"><span class="pre">ContextVar.set()</span></code>, which would represent the modification of
|
||
the context variable in the current logical context:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">var</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'var'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">foo</span><span class="p">():</span>
|
||
<span class="n">mod</span> <span class="o">=</span> <span class="n">var</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'spam'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># ... perform work</span>
|
||
|
||
<span class="n">mod</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span> <span class="c1"># Reset the value of var to the original value</span>
|
||
<span class="c1"># or remove it from the context.</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The critical flaw in this approach is that it becomes possible to
|
||
pass context var “modification objects” into code running in a
|
||
different execution context, which leads to undefined side effects.</p>
|
||
</section>
|
||
</section>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards Compatibility</a></h2>
|
||
<p>This proposal preserves 100% backwards compatibility.</p>
|
||
</section>
|
||
<section id="rejected-ideas">
|
||
<h2><a class="toc-backref" href="#rejected-ideas" role="doc-backlink">Rejected Ideas</a></h2>
|
||
<section id="replication-of-threading-local-interface">
|
||
<h3><a class="toc-backref" href="#replication-of-threading-local-interface" role="doc-backlink">Replication of threading.local() interface</a></h3>
|
||
<p>Choosing the <code class="docutils literal notranslate"><span class="pre">threading.local()</span></code>-like interface for context
|
||
variables was considered and rejected for the following reasons:</p>
|
||
<ul>
|
||
<li>A survey of the standard library and Django has shown that the
|
||
vast majority of <code class="docutils literal notranslate"><span class="pre">threading.local()</span></code> uses involve a single
|
||
attribute, which indicates that the namespace approach is not
|
||
as helpful in the field.</li>
|
||
<li>Using <code class="docutils literal notranslate"><span class="pre">__getattr__()</span></code> instead of <code class="docutils literal notranslate"><span class="pre">.get()</span></code> for value lookup
|
||
does not provide any way to specify the depth of the lookup
|
||
(i.e. search only the top logical context).</li>
|
||
<li>Single-value <code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> is easier to reason about in terms
|
||
of visibility. Suppose <code class="docutils literal notranslate"><span class="pre">ContextVar()</span></code> is a namespace,
|
||
and the consider the following:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ns</span> <span class="o">=</span> <span class="n">contextvars</span><span class="o">.</span><span class="n">ContextVar</span><span class="p">(</span><span class="s1">'ns'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">gen</span><span class="p">():</span>
|
||
<span class="n">ns</span><span class="o">.</span><span class="n">a</span> <span class="o">=</span> <span class="mi">2</span>
|
||
<span class="k">yield</span>
|
||
<span class="k">assert</span> <span class="n">ns</span><span class="o">.</span><span class="n">b</span> <span class="o">==</span> <span class="s1">'bar'</span> <span class="c1"># ??</span>
|
||
|
||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="n">ns</span><span class="o">.</span><span class="n">a</span> <span class="o">=</span> <span class="mi">1</span>
|
||
<span class="n">ns</span><span class="o">.</span><span class="n">b</span> <span class="o">=</span> <span class="s1">'foo'</span>
|
||
<span class="n">g</span> <span class="o">=</span> <span class="n">gen</span><span class="p">()</span>
|
||
<span class="nb">next</span><span class="p">(</span><span class="n">g</span><span class="p">)</span>
|
||
<span class="c1"># should not see the ns.a modification in gen()</span>
|
||
<span class="k">assert</span> <span class="n">ns</span><span class="o">.</span><span class="n">a</span> <span class="o">==</span> <span class="mi">1</span>
|
||
<span class="c1"># but should gen() see the ns.b modification made here?</span>
|
||
<span class="n">ns</span><span class="o">.</span><span class="n">b</span> <span class="o">=</span> <span class="s1">'bar'</span>
|
||
<span class="k">yield</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The above example demonstrates that reasoning about the visibility
|
||
of different attributes of the same context var is not trivial.</p>
|
||
</li>
|
||
<li>Single-value <code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> allows straightforward implementation
|
||
of the lookup cache;</li>
|
||
<li>Single-value <code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> interface allows the C-API to be
|
||
simple and essentially the same as the Python API.</li>
|
||
</ul>
|
||
<p>See also the mailing list discussion: <a class="footnote-reference brackets" href="#id47" id="id10">[26]</a>, <a class="footnote-reference brackets" href="#id48" id="id11">[27]</a>.</p>
|
||
</section>
|
||
<section id="coroutines-not-leaking-context-changes-by-default">
|
||
<h3><a class="toc-backref" href="#coroutines-not-leaking-context-changes-by-default" role="doc-backlink">Coroutines not leaking context changes by default</a></h3>
|
||
<p>In V4 (<a class="reference internal" href="#version-history">Version History</a>) of this PEP, coroutines were considered to
|
||
behave exactly like generators with respect to the execution context:
|
||
changes in awaited coroutines were not visible in the outer coroutine.</p>
|
||
<p>This idea was rejected on the grounds that is breaks the semantic
|
||
similarity of the task and thread models, and, more specifically,
|
||
makes it impossible to reliably implement asynchronous context
|
||
managers that modify context vars, since <code class="docutils literal notranslate"><span class="pre">__aenter__</span></code> is a
|
||
coroutine.</p>
|
||
</section>
|
||
</section>
|
||
<section id="appendix-hamt-performance-analysis">
|
||
<h2><a class="toc-backref" href="#appendix-hamt-performance-analysis" role="doc-backlink">Appendix: HAMT Performance Analysis</a></h2>
|
||
<figure class="align-center" id="id55">
|
||
<a class="invert-in-dark-mode reference internal image-reference" href="../_images/pep-0550-hamt_vs_dict-v2.png"><img alt="../_images/pep-0550-hamt_vs_dict-v2.png" class="invert-in-dark-mode" src="../_images/pep-0550-hamt_vs_dict-v2.png" style="width: 100%;" /></a>
|
||
<figcaption>
|
||
<p><span class="caption-text">Figure 1. Benchmark code can be found here: <a class="footnote-reference brackets" href="#id36" id="id12">[9]</a>.</span></p>
|
||
</figcaption>
|
||
</figure>
|
||
<p>The above chart demonstrates that:</p>
|
||
<ul class="simple">
|
||
<li>HAMT displays near O(1) performance for all benchmarked
|
||
dictionary sizes.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">dict.copy()</span></code> becomes very slow around 100 items.</li>
|
||
</ul>
|
||
<figure class="align-center" id="id56">
|
||
<a class="invert-in-dark-mode reference internal image-reference" href="../_images/pep-0550-lookup_hamt.png"><img alt="../_images/pep-0550-lookup_hamt.png" class="invert-in-dark-mode" src="../_images/pep-0550-lookup_hamt.png" style="width: 100%;" /></a>
|
||
<figcaption>
|
||
<p><span class="caption-text">Figure 2. Benchmark code can be found here: <a class="footnote-reference brackets" href="#id37" id="id13">[10]</a>.</span></p>
|
||
</figcaption>
|
||
</figure>
|
||
<p>Figure 2 compares the lookup costs of <code class="docutils literal notranslate"><span class="pre">dict</span></code> versus a HAMT-based
|
||
immutable mapping. HAMT lookup time is 30-40% slower than Python dict
|
||
lookups on average, which is a very good result, considering that the
|
||
latter is very well optimized.</p>
|
||
<p>There is research <a class="footnote-reference brackets" href="#id35" id="id14">[8]</a> showing that there are further possible
|
||
improvements to the performance of HAMT.</p>
|
||
<p>The reference implementation of HAMT for CPython can be found here:
|
||
<a class="footnote-reference brackets" href="#id34" id="id15">[7]</a>.</p>
|
||
</section>
|
||
<section id="acknowledgments">
|
||
<h2><a class="toc-backref" href="#acknowledgments" role="doc-backlink">Acknowledgments</a></h2>
|
||
<p>Thanks to Victor Petrovykh for countless discussions around the topic
|
||
and PEP proofreading and edits.</p>
|
||
<p>Thanks to Nathaniel Smith for proposing the <code class="docutils literal notranslate"><span class="pre">ContextVar</span></code> design
|
||
<a class="footnote-reference brackets" href="#id38" id="id16">[17]</a> <a class="footnote-reference brackets" href="#id39" id="id17">[18]</a>, for pushing the PEP towards a more complete design, and
|
||
coming up with the idea of having a stack of contexts in the thread
|
||
state.</p>
|
||
<p>Thanks to Alyssa (Nick) Coghlan for numerous suggestions and ideas on the
|
||
mailing list, and for coming up with a case that cause the complete
|
||
rewrite of the initial PEP version <a class="footnote-reference brackets" href="#id40" id="id18">[19]</a>.</p>
|
||
</section>
|
||
<section id="version-history">
|
||
<h2><a class="toc-backref" href="#version-history" role="doc-backlink">Version History</a></h2>
|
||
<ol class="arabic">
|
||
<li>Initial revision, posted on 11-Aug-2017 <a class="footnote-reference brackets" href="#id41" id="id19">[20]</a>.</li>
|
||
<li>V2 posted on 15-Aug-2017 <a class="footnote-reference brackets" href="#id42" id="id20">[21]</a>.<p>The fundamental limitation that caused a complete redesign of the
|
||
first version was that it was not possible to implement an iterator
|
||
that would interact with the EC in the same way as generators
|
||
(see <a class="footnote-reference brackets" href="#id40" id="id21">[19]</a>.)</p>
|
||
<p>Version 2 was a complete rewrite, introducing new terminology
|
||
(Local Context, Execution Context, Context Item) and new APIs.</p>
|
||
</li>
|
||
<li>V3 posted on 18-Aug-2017 <a class="footnote-reference brackets" href="#id43" id="id22">[22]</a>.<p>Updates:</p>
|
||
<ul class="simple">
|
||
<li>Local Context was renamed to Logical Context. The term “local”
|
||
was ambiguous and conflicted with local name scopes.</li>
|
||
<li>Context Item was renamed to Context Key, see the thread with Alyssa
|
||
Coghlan, Stefan Krah, and Yury Selivanov <a class="footnote-reference brackets" href="#id44" id="id23">[23]</a> for details.</li>
|
||
<li>Context Item get cache design was adjusted, per Nathaniel Smith’s
|
||
idea in <a class="footnote-reference brackets" href="#id46" id="id24">[25]</a>.</li>
|
||
<li>Coroutines are created without a Logical Context; ceval loop
|
||
no longer needs to special case the <code class="docutils literal notranslate"><span class="pre">await</span></code> expression
|
||
(proposed by Alyssa Coghlan in <a class="footnote-reference brackets" href="#id45" id="id25">[24]</a>.)</li>
|
||
</ul>
|
||
</li>
|
||
<li>V4 posted on 25-Aug-2017 <a class="footnote-reference brackets" href="#id52" id="id26">[31]</a>.<ul class="simple">
|
||
<li>The specification section has been completely rewritten.</li>
|
||
<li>Coroutines now have their own Logical Context. This means
|
||
there is no difference between coroutines, generators, and
|
||
asynchronous generators w.r.t. interaction with the Execution
|
||
Context.</li>
|
||
<li>Context Key renamed to Context Var.</li>
|
||
<li>Removed the distinction between generators and coroutines with
|
||
respect to logical context isolation.</li>
|
||
</ul>
|
||
</li>
|
||
<li>V5 posted on 01-Sep-2017: the current version.<ul>
|
||
<li>Coroutines have no logical context by default (a revert to the V3
|
||
semantics). Read about the motivation in the
|
||
<a class="reference internal" href="#coroutines-not-leaking-context-changes-by-default">Coroutines not leaking context changes by default</a> section.<p>The <a class="reference internal" href="#high-level-specification">High-Level Specification</a> section was also updated
|
||
(specifically Generators and Coroutines subsections).</p>
|
||
</li>
|
||
<li>All APIs have been placed to the <code class="docutils literal notranslate"><span class="pre">contextvars</span></code> module, and
|
||
the factory functions were changed to class constructors
|
||
(<code class="docutils literal notranslate"><span class="pre">ContextVar</span></code>, <code class="docutils literal notranslate"><span class="pre">ExecutionContext</span></code>, and <code class="docutils literal notranslate"><span class="pre">LogicalContext</span></code>).
|
||
Thanks to Alyssa for the idea <a class="footnote-reference brackets" href="#id54" id="id27">[33]</a>.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">ContextVar.lookup()</span></code> got renamed back to <code class="docutils literal notranslate"><span class="pre">ContextVar.get()</span></code>
|
||
and gained the <code class="docutils literal notranslate"><span class="pre">topmost</span></code> and <code class="docutils literal notranslate"><span class="pre">default</span></code> keyword arguments.
|
||
Added <code class="docutils literal notranslate"><span class="pre">ContextVar.delete()</span></code>.<p>See Guido’s comment in <a class="footnote-reference brackets" href="#id53" id="id28">[32]</a>.</p>
|
||
</li>
|
||
<li>New <code class="docutils literal notranslate"><span class="pre">ExecutionContext.vars()</span></code> method. Read about it in
|
||
the <a class="reference internal" href="#enumerating-context-vars">Enumerating context vars</a> section.</li>
|
||
<li>Fixed <code class="docutils literal notranslate"><span class="pre">ContextVar.get()</span></code> cache bug (thanks Nathaniel!).</li>
|
||
<li>New <a class="reference internal" href="#rejected-ideas">Rejected Ideas</a>,
|
||
<a class="reference internal" href="#should-yield-from-leak-context-changes">Should “yield from” leak context changes?</a>,
|
||
<a class="reference internal" href="#alternative-designs-for-contextvar-api">Alternative Designs for ContextVar API</a>,
|
||
<a class="reference internal" href="#setting-and-restoring-context-variables">Setting and restoring context variables</a>, and
|
||
<a class="reference internal" href="#context-manager-as-the-interface-for-modifications">Context manager as the interface for modifications</a> sections.</li>
|
||
</ul>
|
||
</li>
|
||
</ol>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id29" role="doc-footnote">
|
||
<dt class="label" id="id29">[<a href="#id3">1</a>]</dt>
|
||
<dd><a class="reference external" href="https://go.dev/blog/context">https://go.dev/blog/context</a></aside>
|
||
<aside class="footnote brackets" id="id30" role="doc-footnote">
|
||
<dt class="label" id="id30">[<a href="#id4">2</a>]</dt>
|
||
<dd><a class="reference external" href="https://docs.microsoft.com/en-us/dotnet/api/system.threading.executioncontext">https://docs.microsoft.com/en-us/dotnet/api/system.threading.executioncontext</a></aside>
|
||
<aside class="footnote brackets" id="id31" role="doc-footnote">
|
||
<dt class="label" id="id31">[<a href="#id2">3</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/numpy/numpy/issues/9444">https://github.com/numpy/numpy/issues/9444</a></aside>
|
||
<aside class="footnote brackets" id="id32" role="doc-footnote">
|
||
<dt class="label" id="id32">[<a href="#id6">5</a>]</dt>
|
||
<dd><a class="reference external" href="https://en.wikipedia.org/wiki/Hash_array_mapped_trie">https://en.wikipedia.org/wiki/Hash_array_mapped_trie</a></aside>
|
||
<aside class="footnote brackets" id="id33" role="doc-footnote">
|
||
<dt class="label" id="id33">[<a href="#id7">6</a>]</dt>
|
||
<dd><a class="reference external" href="https://blog.higher-order.net/2010/08/16/assoc-and-clojures-persistenthashmap-part-ii.html">https://blog.higher-order.net/2010/08/16/assoc-and-clojures-persistenthashmap-part-ii.html</a></aside>
|
||
<aside class="footnote brackets" id="id34" role="doc-footnote">
|
||
<dt class="label" id="id34">[<a href="#id15">7</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/1st1/cpython/tree/hamt">https://github.com/1st1/cpython/tree/hamt</a></aside>
|
||
<aside class="footnote brackets" id="id35" role="doc-footnote">
|
||
<dt class="label" id="id35">[<a href="#id14">8</a>]</dt>
|
||
<dd><a class="reference external" href="https://michael.steindorfer.name/publications/oopsla15.pdf">https://michael.steindorfer.name/publications/oopsla15.pdf</a></aside>
|
||
<aside class="footnote brackets" id="id36" role="doc-footnote">
|
||
<dt class="label" id="id36">[<a href="#id12">9</a>]</dt>
|
||
<dd><a class="reference external" href="https://gist.github.com/1st1/9004813d5576c96529527d44c5457dcd">https://gist.github.com/1st1/9004813d5576c96529527d44c5457dcd</a></aside>
|
||
<aside class="footnote brackets" id="id37" role="doc-footnote">
|
||
<dt class="label" id="id37">[<a href="#id13">10</a>]</dt>
|
||
<dd><a class="reference external" href="https://gist.github.com/1st1/dbe27f2e14c30cce6f0b5fddfc8c437e">https://gist.github.com/1st1/dbe27f2e14c30cce6f0b5fddfc8c437e</a></aside>
|
||
<aside class="footnote brackets" id="id38" role="doc-footnote">
|
||
<dt class="label" id="id38">[<a href="#id16">17</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046752.html">https://mail.python.org/pipermail/python-ideas/2017-August/046752.html</a></aside>
|
||
<aside class="footnote brackets" id="id39" role="doc-footnote">
|
||
<dt class="label" id="id39">[<a href="#id17">18</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046772.html">https://mail.python.org/pipermail/python-ideas/2017-August/046772.html</a></aside>
|
||
<aside class="footnote brackets" id="id40" role="doc-footnote">
|
||
<dt class="label" id="id40">[19]<em> (<a href='#id18'>1</a>, <a href='#id21'>2</a>) </em></dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046775.html">https://mail.python.org/pipermail/python-ideas/2017-August/046775.html</a></aside>
|
||
<aside class="footnote brackets" id="id41" role="doc-footnote">
|
||
<dt class="label" id="id41">[<a href="#id19">20</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/python/peps/blob/e8a06c9a790f39451d9e99e203b13b3ad73a1d01/pep-0550.rst">https://github.com/python/peps/blob/e8a06c9a790f39451d9e99e203b13b3ad73a1d01/pep-0550.rst</a></aside>
|
||
<aside class="footnote brackets" id="id42" role="doc-footnote">
|
||
<dt class="label" id="id42">[<a href="#id20">21</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/python/peps/blob/e3aa3b2b4e4e9967d28a10827eed1e9e5960c175/pep-0550.rst">https://github.com/python/peps/blob/e3aa3b2b4e4e9967d28a10827eed1e9e5960c175/pep-0550.rst</a></aside>
|
||
<aside class="footnote brackets" id="id43" role="doc-footnote">
|
||
<dt class="label" id="id43">[<a href="#id22">22</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/python/peps/blob/287ed87bb475a7da657f950b353c71c1248f67e7/pep-0550.rst">https://github.com/python/peps/blob/287ed87bb475a7da657f950b353c71c1248f67e7/pep-0550.rst</a></aside>
|
||
<aside class="footnote brackets" id="id44" role="doc-footnote">
|
||
<dt class="label" id="id44">[<a href="#id23">23</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046801.html">https://mail.python.org/pipermail/python-ideas/2017-August/046801.html</a></aside>
|
||
<aside class="footnote brackets" id="id45" role="doc-footnote">
|
||
<dt class="label" id="id45">[<a href="#id25">24</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046790.html">https://mail.python.org/pipermail/python-ideas/2017-August/046790.html</a></aside>
|
||
<aside class="footnote brackets" id="id46" role="doc-footnote">
|
||
<dt class="label" id="id46">[<a href="#id24">25</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046786.html">https://mail.python.org/pipermail/python-ideas/2017-August/046786.html</a></aside>
|
||
<aside class="footnote brackets" id="id47" role="doc-footnote">
|
||
<dt class="label" id="id47">[<a href="#id10">26</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046888.html">https://mail.python.org/pipermail/python-ideas/2017-August/046888.html</a></aside>
|
||
<aside class="footnote brackets" id="id48" role="doc-footnote">
|
||
<dt class="label" id="id48">[<a href="#id11">27</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-August/046889.html">https://mail.python.org/pipermail/python-ideas/2017-August/046889.html</a></aside>
|
||
<aside class="footnote brackets" id="id49" role="doc-footnote">
|
||
<dt class="label" id="id49">[<a href="#id1">28</a>]</dt>
|
||
<dd><a class="reference external" href="https://docs.python.org/3/library/decimal.html#decimal.Context.abs">https://docs.python.org/3/library/decimal.html#decimal.Context.abs</a></aside>
|
||
<aside class="footnote brackets" id="id50" role="doc-footnote">
|
||
<dt class="label" id="id50">[<a href="#id8">29</a>]</dt>
|
||
<dd><a class="reference external" href="https://web.archive.org/web/20170706074739/https://curio.readthedocs.io/en/latest/reference.html#task-local-storage">https://web.archive.org/web/20170706074739/https://curio.readthedocs.io/en/latest/reference.html#task-local-storage</a></aside>
|
||
<aside class="footnote brackets" id="id51" role="doc-footnote">
|
||
<dt class="label" id="id51">[<a href="#id9">30</a>]</dt>
|
||
<dd><a class="reference external" href="https://docs.atlassian.com/aiolocals/latest/usage.html">https://docs.atlassian.com/aiolocals/latest/usage.html</a></aside>
|
||
<aside class="footnote brackets" id="id52" role="doc-footnote">
|
||
<dt class="label" id="id52">[<a href="#id26">31</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/python/peps/blob/1b8728ded7cde9df0f9a24268574907fafec6d5e/pep-0550.rst">https://github.com/python/peps/blob/1b8728ded7cde9df0f9a24268574907fafec6d5e/pep-0550.rst</a></aside>
|
||
<aside class="footnote brackets" id="id53" role="doc-footnote">
|
||
<dt class="label" id="id53">[<a href="#id28">32</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-dev/2017-August/149020.html">https://mail.python.org/pipermail/python-dev/2017-August/149020.html</a></aside>
|
||
<aside class="footnote brackets" id="id54" role="doc-footnote">
|
||
<dt class="label" id="id54">[<a href="#id27">33</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-dev/2017-August/149043.html">https://mail.python.org/pipermail/python-dev/2017-August/149043.html</a></aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0550.rst">https://github.com/python/peps/blob/main/peps/pep-0550.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0550.rst">2023-10-11 12:05:51 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#pep-status">PEP Status</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#goals">Goals</a></li>
|
||
<li><a class="reference internal" href="#high-level-specification">High-Level Specification</a><ul>
|
||
<li><a class="reference internal" href="#regular-single-threaded-code">Regular Single-threaded Code</a></li>
|
||
<li><a class="reference internal" href="#multithreaded-code">Multithreaded Code</a></li>
|
||
<li><a class="reference internal" href="#generators">Generators</a></li>
|
||
<li><a class="reference internal" href="#coroutines-and-asynchronous-tasks">Coroutines and Asynchronous Tasks</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#detailed-specification">Detailed Specification</a><ul>
|
||
<li><a class="reference internal" href="#id5">Generators</a></li>
|
||
<li><a class="reference internal" href="#contextlib-contextmanager">contextlib.contextmanager</a></li>
|
||
<li><a class="reference internal" href="#enumerating-context-vars">Enumerating context vars</a></li>
|
||
<li><a class="reference internal" href="#coroutines">coroutines</a></li>
|
||
<li><a class="reference internal" href="#asynchronous-generators">Asynchronous Generators</a></li>
|
||
<li><a class="reference internal" href="#asyncio">asyncio</a></li>
|
||
<li><a class="reference internal" href="#generators-transformed-into-iterators">Generators Transformed into Iterators</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a><ul>
|
||
<li><a class="reference internal" href="#logical-context">Logical Context</a></li>
|
||
<li><a class="reference internal" href="#context-variables">Context Variables</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#performance-considerations">Performance Considerations</a></li>
|
||
<li><a class="reference internal" href="#summary-of-the-new-apis">Summary of the New APIs</a><ul>
|
||
<li><a class="reference internal" href="#python">Python</a></li>
|
||
<li><a class="reference internal" href="#c-api">C API</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#design-considerations">Design Considerations</a><ul>
|
||
<li><a class="reference internal" href="#should-yield-from-leak-context-changes">Should “yield from” leak context changes?</a></li>
|
||
<li><a class="reference internal" href="#should-pythreadstate-getdict-use-the-execution-context">Should <code class="docutils literal notranslate"><span class="pre">PyThreadState_GetDict()</span></code> use the execution context?</a></li>
|
||
<li><a class="reference internal" href="#pep-521">PEP 521</a></li>
|
||
<li><a class="reference internal" href="#can-execution-context-be-implemented-without-modifying-cpython">Can Execution Context be implemented without modifying CPython?</a></li>
|
||
<li><a class="reference internal" href="#should-we-update-sys-displayhook-and-other-apis-to-use-ec">Should we update sys.displayhook and other APIs to use EC?</a></li>
|
||
<li><a class="reference internal" href="#greenlets">Greenlets</a></li>
|
||
<li><a class="reference internal" href="#context-manager-as-the-interface-for-modifications">Context manager as the interface for modifications</a></li>
|
||
<li><a class="reference internal" href="#setting-and-restoring-context-variables">Setting and restoring context variables</a></li>
|
||
<li><a class="reference internal" href="#alternative-designs-for-contextvar-api">Alternative Designs for ContextVar API</a><ul>
|
||
<li><a class="reference internal" href="#logical-context-with-stacked-values">Logical Context with stacked values</a></li>
|
||
<li><a class="reference internal" href="#contextvar-set-reset">ContextVar “set/reset”</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#replication-of-threading-local-interface">Replication of threading.local() interface</a></li>
|
||
<li><a class="reference internal" href="#coroutines-not-leaking-context-changes-by-default">Coroutines not leaking context changes by default</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#appendix-hamt-performance-analysis">Appendix: HAMT Performance Analysis</a></li>
|
||
<li><a class="reference internal" href="#acknowledgments">Acknowledgments</a></li>
|
||
<li><a class="reference internal" href="#version-history">Version History</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0550.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |