mirror of https://github.com/python/peps
593 lines
46 KiB
HTML
593 lines
46 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 3148 – futures - execute computations asynchronously | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-3148/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 3148 – futures - execute computations asynchronously | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-3148/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 3148</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 3148 – futures - execute computations asynchronously</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Brian Quinlan <brian at sweetapp.com></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Accepted and implementation complete, or no longer active">Final</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">16-Oct-2009</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.2</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p></p></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#naming">Naming</a></li>
|
||
<li><a class="reference internal" href="#interface">Interface</a><ul>
|
||
<li><a class="reference internal" href="#executor">Executor</a></li>
|
||
<li><a class="reference internal" href="#processpoolexecutor">ProcessPoolExecutor</a></li>
|
||
<li><a class="reference internal" href="#threadpoolexecutor">ThreadPoolExecutor</a></li>
|
||
<li><a class="reference internal" href="#future-objects">Future Objects</a><ul>
|
||
<li><a class="reference internal" href="#internal-future-methods">Internal Future Methods</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#module-functions">Module Functions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#check-prime-example">Check Prime Example</a></li>
|
||
<li><a class="reference internal" href="#web-crawl-example">Web Crawl Example</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP proposes a design for a package that facilitates the
|
||
evaluation of callables using threads and processes.</p>
|
||
</section>
|
||
<section id="motivation">
|
||
<h2><a class="toc-backref" href="#motivation" role="doc-backlink">Motivation</a></h2>
|
||
<p>Python currently has powerful primitives to construct multi-threaded
|
||
and multi-process applications but parallelizing simple operations
|
||
requires a lot of work i.e. explicitly launching processes/threads,
|
||
constructing a work/results queue, and waiting for completion or some
|
||
other termination condition (e.g. failure, timeout). It is also
|
||
difficult to design an application with a global process/thread limit
|
||
when each component invents its own parallel execution strategy.</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<section id="naming">
|
||
<h3><a class="toc-backref" href="#naming" role="doc-backlink">Naming</a></h3>
|
||
<p>The proposed package would be called “futures” and would live in a new
|
||
“concurrent” top-level package. The rationale behind pushing the
|
||
futures library into a “concurrent” namespace has multiple components.
|
||
The first, most simple one is to prevent any and all confusion with
|
||
the existing “from __future__ import x” idiom which has been in use
|
||
for a long time within Python. Additionally, it is felt that adding
|
||
the “concurrent” precursor to the name fully denotes what the library
|
||
is related to - namely concurrency - this should clear up any addition
|
||
ambiguity as it has been noted that not everyone in the community is
|
||
familiar with Java Futures, or the Futures term except as it relates
|
||
to the US stock market.</p>
|
||
<p>Finally; we are carving out a new namespace for the standard library -
|
||
obviously named “concurrent”. We hope to either add, or move existing,
|
||
concurrency-related libraries to this in the future. A prime example
|
||
is the multiprocessing.Pool work, as well as other “addons” included
|
||
in that module, which work across thread and process boundaries.</p>
|
||
</section>
|
||
<section id="interface">
|
||
<h3><a class="toc-backref" href="#interface" role="doc-backlink">Interface</a></h3>
|
||
<p>The proposed package provides two core classes: <code class="docutils literal notranslate"><span class="pre">Executor</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">Future</span></code>. An <code class="docutils literal notranslate"><span class="pre">Executor</span></code> receives asynchronous work requests (in terms
|
||
of a callable and its arguments) and returns a <code class="docutils literal notranslate"><span class="pre">Future</span></code> to represent
|
||
the execution of that work request.</p>
|
||
<section id="executor">
|
||
<h4><a class="toc-backref" href="#executor" role="doc-backlink">Executor</a></h4>
|
||
<p><code class="docutils literal notranslate"><span class="pre">Executor</span></code> is an abstract class that provides methods to execute calls
|
||
asynchronously.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">submit(fn,</span> <span class="pre">*args,</span> <span class="pre">**kwargs)</span></code></p>
|
||
<blockquote>
|
||
<div>Schedules the callable to be executed as <code class="docutils literal notranslate"><span class="pre">fn(*args,</span> <span class="pre">**kwargs)</span></code>
|
||
and returns a <code class="docutils literal notranslate"><span class="pre">Future</span></code> instance representing the execution of the
|
||
callable.<p>This is an abstract method and must be implemented by Executor
|
||
subclasses.</p>
|
||
</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">map(func,</span> <span class="pre">*iterables,</span> <span class="pre">timeout=None)</span></code></p>
|
||
<blockquote>
|
||
<div>Equivalent to <code class="docutils literal notranslate"><span class="pre">map(func,</span> <span class="pre">*iterables)</span></code> but func is executed
|
||
asynchronously and several calls to func may be made concurrently.
|
||
The returned iterator raises a <code class="docutils literal notranslate"><span class="pre">TimeoutError</span></code> if <code class="docutils literal notranslate"><span class="pre">__next__()</span></code> is
|
||
called and the result isn’t available after <em>timeout</em> seconds from
|
||
the original call to <code class="docutils literal notranslate"><span class="pre">map()</span></code>. If <em>timeout</em> is not specified or
|
||
<code class="docutils literal notranslate"><span class="pre">None</span></code> then there is no limit to the wait time. If a call raises
|
||
an exception then that exception will be raised when its value is
|
||
retrieved from the iterator.</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">shutdown(wait=True)</span></code></p>
|
||
<blockquote>
|
||
<div>Signal the executor that it should free any resources that it is
|
||
using when the currently pending futures are done executing.
|
||
Calls to <code class="docutils literal notranslate"><span class="pre">Executor.submit</span></code> and <code class="docutils literal notranslate"><span class="pre">Executor.map</span></code> and made after
|
||
shutdown will raise <code class="docutils literal notranslate"><span class="pre">RuntimeError</span></code>.<p>If wait is <code class="docutils literal notranslate"><span class="pre">True</span></code> then this method will not return until all the
|
||
pending futures are done executing and the resources associated
|
||
with the executor have been freed. If wait is <code class="docutils literal notranslate"><span class="pre">False</span></code> then this
|
||
method will return immediately and the resources associated with
|
||
the executor will be freed when all pending futures are done
|
||
executing. Regardless of the value of wait, the entire Python
|
||
program will not exit until all pending futures are done
|
||
executing.</p>
|
||
</div></blockquote>
|
||
<div class="line-block">
|
||
<div class="line"><code class="docutils literal notranslate"><span class="pre">__enter__()</span></code></div>
|
||
<div class="line"><code class="docutils literal notranslate"><span class="pre">__exit__(exc_type,</span> <span class="pre">exc_val,</span> <span class="pre">exc_tb)</span></code></div>
|
||
</div>
|
||
<blockquote>
|
||
<div>When using an executor as a context manager, <code class="docutils literal notranslate"><span class="pre">__exit__</span></code> will call
|
||
<code class="docutils literal notranslate"><span class="pre">Executor.shutdown(wait=True)</span></code>.</div></blockquote>
|
||
</section>
|
||
<section id="processpoolexecutor">
|
||
<h4><a class="toc-backref" href="#processpoolexecutor" role="doc-backlink">ProcessPoolExecutor</a></h4>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">ProcessPoolExecutor</span></code> class is an <code class="docutils literal notranslate"><span class="pre">Executor</span></code> subclass that uses a
|
||
pool of processes to execute calls asynchronously. The callable
|
||
objects and arguments passed to <code class="docutils literal notranslate"><span class="pre">ProcessPoolExecutor.submit</span></code> must be
|
||
pickleable according to the same limitations as the multiprocessing
|
||
module.</p>
|
||
<p>Calling <code class="docutils literal notranslate"><span class="pre">Executor</span></code> or <code class="docutils literal notranslate"><span class="pre">Future</span></code> methods from within a callable
|
||
submitted to a <code class="docutils literal notranslate"><span class="pre">ProcessPoolExecutor</span></code> will result in deadlock.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">__init__(max_workers)</span></code></p>
|
||
<blockquote>
|
||
<div>Executes calls asynchronously using a pool of a most <em>max_workers</em>
|
||
processes. If <em>max_workers</em> is <code class="docutils literal notranslate"><span class="pre">None</span></code> or not given then as many
|
||
worker processes will be created as the machine has processors.</div></blockquote>
|
||
</section>
|
||
<section id="threadpoolexecutor">
|
||
<h4><a class="toc-backref" href="#threadpoolexecutor" role="doc-backlink">ThreadPoolExecutor</a></h4>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> class is an <code class="docutils literal notranslate"><span class="pre">Executor</span></code> subclass that uses a
|
||
pool of threads to execute calls asynchronously.</p>
|
||
<p>Deadlock can occur when the callable associated with a <code class="docutils literal notranslate"><span class="pre">Future</span></code> waits
|
||
on the results of another <code class="docutils literal notranslate"><span class="pre">Future</span></code>. For example:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">time</span>
|
||
<span class="k">def</span> <span class="nf">wait_on_b</span><span class="p">():</span>
|
||
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">b</span><span class="o">.</span><span class="n">result</span><span class="p">())</span> <span class="c1"># b will never complete because it is waiting on a.</span>
|
||
<span class="k">return</span> <span class="mi">5</span>
|
||
|
||
<span class="k">def</span> <span class="nf">wait_on_a</span><span class="p">():</span>
|
||
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">result</span><span class="p">())</span> <span class="c1"># a will never complete because it is waiting on b.</span>
|
||
<span class="k">return</span> <span class="mi">6</span>
|
||
|
||
|
||
<span class="n">executor</span> <span class="o">=</span> <span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">wait_on_b</span><span class="p">)</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">wait_on_a</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>And:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">wait_on_future</span><span class="p">():</span>
|
||
<span class="n">f</span> <span class="o">=</span> <span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="nb">pow</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="c1"># This will never complete because there is only one worker thread and</span>
|
||
<span class="c1"># it is executing this function.</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">result</span><span class="p">())</span>
|
||
|
||
<span class="n">executor</span> <span class="o">=</span> <span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">wait_on_future</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><code class="docutils literal notranslate"><span class="pre">__init__(max_workers)</span></code></p>
|
||
<blockquote>
|
||
<div>Executes calls asynchronously using a pool of at most
|
||
<em>max_workers</em> threads.</div></blockquote>
|
||
</section>
|
||
<section id="future-objects">
|
||
<h4><a class="toc-backref" href="#future-objects" role="doc-backlink">Future Objects</a></h4>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">Future</span></code> class encapsulates the asynchronous execution of a
|
||
callable. <code class="docutils literal notranslate"><span class="pre">Future</span></code> instances are returned by <code class="docutils literal notranslate"><span class="pre">Executor.submit</span></code>.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">cancel()</span></code></p>
|
||
<blockquote>
|
||
<div>Attempt to cancel the call. If the call is currently being
|
||
executed then it cannot be cancelled and the method will return
|
||
<code class="docutils literal notranslate"><span class="pre">False</span></code>, otherwise the call will be cancelled and the method will
|
||
return <code class="docutils literal notranslate"><span class="pre">True</span></code>.</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">cancelled()</span></code></p>
|
||
<blockquote>
|
||
<div>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> if the call was successfully cancelled.</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">running()</span></code></p>
|
||
<blockquote>
|
||
<div>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> if the call is currently being executed and cannot
|
||
be cancelled.</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">done()</span></code></p>
|
||
<blockquote>
|
||
<div>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> if the call was successfully cancelled or finished
|
||
running.</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">result(timeout=None)</span></code></p>
|
||
<blockquote>
|
||
<div>Return the value returned by the call. If the call hasn’t yet
|
||
completed then this method will wait up to <em>timeout</em> seconds. If
|
||
the call hasn’t completed in <em>timeout</em> seconds then a
|
||
<code class="docutils literal notranslate"><span class="pre">TimeoutError</span></code> will be raised. If <em>timeout</em> is not specified or
|
||
<code class="docutils literal notranslate"><span class="pre">None</span></code> then there is no limit to the wait time.<p>If the future is cancelled before completing then <code class="docutils literal notranslate"><span class="pre">CancelledError</span></code>
|
||
will be raised.</p>
|
||
<p>If the call raised then this method will raise the same exception.</p>
|
||
</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">exception(timeout=None)</span></code></p>
|
||
<blockquote>
|
||
<div>Return the exception raised by the call. If the call hasn’t yet
|
||
completed then this method will wait up to <em>timeout</em> seconds. If
|
||
the call hasn’t completed in <em>timeout</em> seconds then a
|
||
<code class="docutils literal notranslate"><span class="pre">TimeoutError</span></code> will be raised. If <em>timeout</em> is not specified or
|
||
<code class="docutils literal notranslate"><span class="pre">None</span></code> then there is no limit to the wait time.<p>If the future is cancelled before completing then <code class="docutils literal notranslate"><span class="pre">CancelledError</span></code>
|
||
will be raised.</p>
|
||
<p>If the call completed without raising then <code class="docutils literal notranslate"><span class="pre">None</span></code> is returned.</p>
|
||
</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">add_done_callback(fn)</span></code></p>
|
||
<blockquote>
|
||
<div>Attaches a callable <em>fn</em> to the future that will be called when
|
||
the future is cancelled or finishes running. <em>fn</em> will be called
|
||
with the future as its only argument.<p>Added callables are called in the order that they were added and
|
||
are always called in a thread belonging to the process that added
|
||
them. If the callable raises an <code class="docutils literal notranslate"><span class="pre">Exception</span></code> then it will be
|
||
logged and ignored. If the callable raises another
|
||
<code class="docutils literal notranslate"><span class="pre">BaseException</span></code> then behavior is not defined.</p>
|
||
<p>If the future has already completed or been cancelled then <em>fn</em>
|
||
will be called immediately.</p>
|
||
</div></blockquote>
|
||
<section id="internal-future-methods">
|
||
<h5><a class="toc-backref" href="#internal-future-methods" role="doc-backlink">Internal Future Methods</a></h5>
|
||
<p>The following <code class="docutils literal notranslate"><span class="pre">Future</span></code> methods are meant for use in unit tests and
|
||
<code class="docutils literal notranslate"><span class="pre">Executor</span></code> implementations.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">set_running_or_notify_cancel()</span></code></p>
|
||
<blockquote>
|
||
<div>Should be called by <code class="docutils literal notranslate"><span class="pre">Executor</span></code> implementations before executing
|
||
the work associated with the <code class="docutils literal notranslate"><span class="pre">Future</span></code>.<p>If the method returns <code class="docutils literal notranslate"><span class="pre">False</span></code> then the <code class="docutils literal notranslate"><span class="pre">Future</span></code> was cancelled,
|
||
i.e. <code class="docutils literal notranslate"><span class="pre">Future.cancel</span></code> was called and returned <code class="docutils literal notranslate"><span class="pre">True</span></code>. Any threads
|
||
waiting on the <code class="docutils literal notranslate"><span class="pre">Future</span></code> completing (i.e. through <code class="docutils literal notranslate"><span class="pre">as_completed()</span></code>
|
||
or <code class="docutils literal notranslate"><span class="pre">wait()</span></code>) will be woken up.</p>
|
||
<p>If the method returns <code class="docutils literal notranslate"><span class="pre">True</span></code> then the <code class="docutils literal notranslate"><span class="pre">Future</span></code> was not cancelled
|
||
and has been put in the running state, i.e. calls to
|
||
<code class="docutils literal notranslate"><span class="pre">Future.running()</span></code> will return <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
|
||
<p>This method can only be called once and cannot be called after
|
||
<code class="docutils literal notranslate"><span class="pre">Future.set_result()</span></code> or <code class="docutils literal notranslate"><span class="pre">Future.set_exception()</span></code> have been
|
||
called.</p>
|
||
</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">set_result(result)</span></code></p>
|
||
<blockquote>
|
||
<div>Sets the result of the work associated with the <code class="docutils literal notranslate"><span class="pre">Future</span></code>.</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">set_exception(exception)</span></code></p>
|
||
<blockquote>
|
||
<div>Sets the result of the work associated with the <code class="docutils literal notranslate"><span class="pre">Future</span></code> to the
|
||
given <code class="docutils literal notranslate"><span class="pre">Exception</span></code>.</div></blockquote>
|
||
</section>
|
||
</section>
|
||
<section id="module-functions">
|
||
<h4><a class="toc-backref" href="#module-functions" role="doc-backlink">Module Functions</a></h4>
|
||
<p><code class="docutils literal notranslate"><span class="pre">wait(fs,</span> <span class="pre">timeout=None,</span> <span class="pre">return_when=ALL_COMPLETED)</span></code></p>
|
||
<blockquote>
|
||
<div>Wait for the <code class="docutils literal notranslate"><span class="pre">Future</span></code> instances (possibly created by different
|
||
<code class="docutils literal notranslate"><span class="pre">Executor</span></code> instances) given by <em>fs</em> to complete. Returns a named
|
||
2-tuple of sets. The first set, named “done”, contains the
|
||
futures that completed (finished or were cancelled) before the
|
||
wait completed. The second set, named “not_done”, contains
|
||
uncompleted futures.<p><em>timeout</em> can be used to control the maximum number of seconds to
|
||
wait before returning. If timeout is not specified or None then
|
||
there is no limit to the wait time.</p>
|
||
<p><em>return_when</em> indicates when the method should return. It must be
|
||
one of the following constants:</p>
|
||
<table class="docutils align-default">
|
||
<thead>
|
||
<tr class="row-odd"><th class="head">Constant</th>
|
||
<th class="head">Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">FIRST_COMPLETED</span></code></td>
|
||
<td>The method will return when any future finishes or
|
||
is cancelled.</td>
|
||
</tr>
|
||
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">FIRST_EXCEPTION</span></code></td>
|
||
<td>The method will return when any future finishes by
|
||
raising an exception. If not future raises an
|
||
exception then it is equivalent to ALL_COMPLETED.</td>
|
||
</tr>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">ALL_COMPLETED</span></code></td>
|
||
<td>The method will return when all calls finish.</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div></blockquote>
|
||
<p><code class="docutils literal notranslate"><span class="pre">as_completed(fs,</span> <span class="pre">timeout=None)</span></code></p>
|
||
<blockquote>
|
||
<div>Returns an iterator over the <code class="docutils literal notranslate"><span class="pre">Future</span></code> instances given by <em>fs</em> that
|
||
yields futures as they complete (finished or were cancelled). Any
|
||
futures that completed before <code class="docutils literal notranslate"><span class="pre">as_completed()</span></code> was called will be
|
||
yielded first. The returned iterator raises a <code class="docutils literal notranslate"><span class="pre">TimeoutError</span></code> if
|
||
<code class="docutils literal notranslate"><span class="pre">__next__()</span></code> is called and the result isn’t available after
|
||
<em>timeout</em> seconds from the original call to <code class="docutils literal notranslate"><span class="pre">as_completed()</span></code>. If
|
||
<em>timeout</em> is not specified or <code class="docutils literal notranslate"><span class="pre">None</span></code> then there is no limit to the
|
||
wait time.<p>The <code class="docutils literal notranslate"><span class="pre">Future</span></code> instances can have been created by different
|
||
<code class="docutils literal notranslate"><span class="pre">Executor</span></code> instances.</p>
|
||
</div></blockquote>
|
||
</section>
|
||
</section>
|
||
<section id="check-prime-example">
|
||
<h3><a class="toc-backref" href="#check-prime-example" role="doc-backlink">Check Prime Example</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">concurrent</span> <span class="kn">import</span> <span class="n">futures</span>
|
||
<span class="kn">import</span> <span class="nn">math</span>
|
||
|
||
<span class="n">PRIMES</span> <span class="o">=</span> <span class="p">[</span>
|
||
<span class="mi">112272535095293</span><span class="p">,</span>
|
||
<span class="mi">112582705942171</span><span class="p">,</span>
|
||
<span class="mi">112272535095293</span><span class="p">,</span>
|
||
<span class="mi">115280095190773</span><span class="p">,</span>
|
||
<span class="mi">115797848077099</span><span class="p">,</span>
|
||
<span class="mi">1099726899285419</span><span class="p">]</span>
|
||
|
||
<span class="k">def</span> <span class="nf">is_prime</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">n</span> <span class="o">%</span> <span class="mi">2</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="kc">False</span>
|
||
|
||
<span class="n">sqrt_n</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">floor</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">n</span><span class="p">)))</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">sqrt_n</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">n</span> <span class="o">%</span> <span class="n">i</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="kc">False</span>
|
||
<span class="k">return</span> <span class="kc">True</span>
|
||
|
||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">futures</span><span class="o">.</span><span class="n">ProcessPoolExecutor</span><span class="p">()</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
|
||
<span class="k">for</span> <span class="n">number</span><span class="p">,</span> <span class="n">prime</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">PRIMES</span><span class="p">,</span> <span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">is_prime</span><span class="p">,</span>
|
||
<span class="n">PRIMES</span><span class="p">)):</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'</span><span class="si">%d</span><span class="s1"> is prime: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">number</span><span class="p">,</span> <span class="n">prime</span><span class="p">))</span>
|
||
|
||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||
<span class="n">main</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="web-crawl-example">
|
||
<h3><a class="toc-backref" href="#web-crawl-example" role="doc-backlink">Web Crawl Example</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">concurrent</span> <span class="kn">import</span> <span class="n">futures</span>
|
||
<span class="kn">import</span> <span class="nn">urllib.request</span>
|
||
|
||
<span class="n">URLS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'http://www.foxnews.com/'</span><span class="p">,</span>
|
||
<span class="s1">'http://www.cnn.com/'</span><span class="p">,</span>
|
||
<span class="s1">'http://europe.wsj.com/'</span><span class="p">,</span>
|
||
<span class="s1">'http://www.bbc.co.uk/'</span><span class="p">,</span>
|
||
<span class="s1">'http://some-made-up-domain.com/'</span><span class="p">]</span>
|
||
|
||
<span class="k">def</span> <span class="nf">load_url</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">timeout</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">urllib</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
||
|
||
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
|
||
<span class="k">with</span> <span class="n">futures</span><span class="o">.</span><span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
|
||
<span class="n">future_to_url</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||
<span class="p">(</span><span class="n">executor</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">load_url</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="mi">60</span><span class="p">),</span> <span class="n">url</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">URLS</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="o">.</span><span class="n">as_completed</span><span class="p">(</span><span class="n">future_to_url</span><span class="p">):</span>
|
||
<span class="n">url</span> <span class="o">=</span> <span class="n">future_to_url</span><span class="p">[</span><span class="n">future</span><span class="p">]</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'</span><span class="si">%r</span><span class="s1"> page is </span><span class="si">%d</span><span class="s1"> bytes'</span> <span class="o">%</span> <span class="p">(</span>
|
||
<span class="n">url</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">future</span><span class="o">.</span><span class="n">result</span><span class="p">())))</span>
|
||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'</span><span class="si">%r</span><span class="s1"> generated an exception: </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span>
|
||
<span class="n">url</span><span class="p">,</span> <span class="n">e</span><span class="p">))</span>
|
||
|
||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||
<span class="n">main</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>The proposed design of this module was heavily influenced by the
|
||
Java java.util.concurrent package <a class="footnote-reference brackets" href="#id9" id="id1">[1]</a>. The conceptual basis of the
|
||
module, as in Java, is the Future class, which represents the progress
|
||
and result of an asynchronous computation. The Future class makes
|
||
little commitment to the evaluation mode being used e.g. it can be
|
||
used to represent lazy or eager evaluation, for evaluation using
|
||
threads, processes or remote procedure call.</p>
|
||
<p>Futures are created by concrete implementations of the Executor class
|
||
(called ExecutorService in Java). The reference implementation
|
||
provides classes that use either a process or a thread pool to eagerly
|
||
evaluate computations.</p>
|
||
<p>Futures have already been seen in Python as part of a popular Python
|
||
cookbook recipe <a class="footnote-reference brackets" href="#id10" id="id2">[2]</a> and have discussed on the Python-3000 mailing
|
||
list <a class="footnote-reference brackets" href="#id11" id="id3">[3]</a>.</p>
|
||
<p>The proposed design is explicit, i.e. it requires that clients be
|
||
aware that they are consuming Futures. It would be possible to design
|
||
a module that would return proxy objects (in the style of <code class="docutils literal notranslate"><span class="pre">weakref</span></code>)
|
||
that could be used transparently. It is possible to build a proxy
|
||
implementation on top of the proposed explicit mechanism.</p>
|
||
<p>The proposed design does not introduce any changes to Python language
|
||
syntax or semantics. Special syntax could be introduced <a class="footnote-reference brackets" href="#id12" id="id4">[4]</a> to mark
|
||
function and method calls as asynchronous. A proxy result would be
|
||
returned while the operation is eagerly evaluated asynchronously, and
|
||
execution would only block if the proxy object were used before the
|
||
operation completed.</p>
|
||
<p>Anh Hai Trinh proposed a simpler but more limited API concept <a class="footnote-reference brackets" href="#id13" id="id5">[5]</a> and
|
||
the API has been discussed in some detail on stdlib-sig <a class="footnote-reference brackets" href="#id14" id="id6">[6]</a>.</p>
|
||
<p>The proposed design was discussed on the Python-Dev mailing list <a class="footnote-reference brackets" href="#id15" id="id7">[7]</a>.
|
||
Following those discussions, the following changes were made:</p>
|
||
<ul class="simple">
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">Executor</span></code> class was made into an abstract base class</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">Future.remove_done_callback</span></code> method was removed due to a lack
|
||
of convincing use cases</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">Future.add_done_callback</span></code> method was modified to allow the
|
||
same callable to be added many times</li>
|
||
<li>The <code class="docutils literal notranslate"><span class="pre">Future</span></code> class’s mutation methods were better documented to
|
||
indicate that they are private to the <code class="docutils literal notranslate"><span class="pre">Executor</span></code> that created them</li>
|
||
</ul>
|
||
</section>
|
||
<section id="reference-implementation">
|
||
<h2><a class="toc-backref" href="#reference-implementation" role="doc-backlink">Reference Implementation</a></h2>
|
||
<p>The reference implementation <a class="footnote-reference brackets" href="#id16" id="id8">[8]</a> contains a complete implementation
|
||
of the proposed design. It has been tested on Linux and Mac OS X.</p>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id9" role="doc-footnote">
|
||
<dt class="label" id="id9">[<a href="#id1">1</a>]</dt>
|
||
<dd><code class="docutils literal notranslate"><span class="pre">java.util.concurrent</span></code> package documentation
|
||
<a class="reference external" href="http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/package-summary.html">http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/package-summary.html</a></aside>
|
||
<aside class="footnote brackets" id="id10" role="doc-footnote">
|
||
<dt class="label" id="id10">[<a href="#id2">2</a>]</dt>
|
||
<dd>Python Cookbook recipe 84317, “Easy threading with Futures”
|
||
<a class="reference external" href="http://code.activestate.com/recipes/84317/">http://code.activestate.com/recipes/84317/</a></aside>
|
||
<aside class="footnote brackets" id="id11" role="doc-footnote">
|
||
<dt class="label" id="id11">[<a href="#id3">3</a>]</dt>
|
||
<dd><code class="docutils literal notranslate"><span class="pre">Python-3000</span></code> thread, “mechanism for handling asynchronous concurrency”
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-3000/2006-April/000960.html">https://mail.python.org/pipermail/python-3000/2006-April/000960.html</a></aside>
|
||
<aside class="footnote brackets" id="id12" role="doc-footnote">
|
||
<dt class="label" id="id12">[<a href="#id4">4</a>]</dt>
|
||
<dd><code class="docutils literal notranslate"><span class="pre">Python</span> <span class="pre">3000</span></code> thread, “Futures in Python 3000 (was Re: mechanism for handling asynchronous concurrency)”
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-3000/2006-April/000970.html">https://mail.python.org/pipermail/python-3000/2006-April/000970.html</a></aside>
|
||
<aside class="footnote brackets" id="id13" role="doc-footnote">
|
||
<dt class="label" id="id13">[<a href="#id5">5</a>]</dt>
|
||
<dd>A discussion of <code class="docutils literal notranslate"><span class="pre">stream</span></code>, a similar concept proposed by Anh Hai Trinh
|
||
<a class="reference external" href="http://www.mail-archive.com/stdlib-sig@python.org/msg00480.html">http://www.mail-archive.com/stdlib-sig@python.org/msg00480.html</a></aside>
|
||
<aside class="footnote brackets" id="id14" role="doc-footnote">
|
||
<dt class="label" id="id14">[<a href="#id6">6</a>]</dt>
|
||
<dd>A discussion of the proposed API on stdlib-sig
|
||
<a class="reference external" href="https://mail.python.org/pipermail/stdlib-sig/2009-November/000731.html">https://mail.python.org/pipermail/stdlib-sig/2009-November/000731.html</a></aside>
|
||
<aside class="footnote brackets" id="id15" role="doc-footnote">
|
||
<dt class="label" id="id15">[<a href="#id7">7</a>]</dt>
|
||
<dd>A discussion of the PEP on python-dev
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-dev/2010-March/098169.html">https://mail.python.org/pipermail/python-dev/2010-March/098169.html</a></aside>
|
||
<aside class="footnote brackets" id="id16" role="doc-footnote">
|
||
<dt class="label" id="id16">[<a href="#id8">8</a>]</dt>
|
||
<dd>Reference <code class="docutils literal notranslate"><span class="pre">futures</span></code> implementation
|
||
<a class="reference external" href="http://code.google.com/p/pythonfutures/source/browse/#svn/branches/feedback">http://code.google.com/p/pythonfutures/source/browse/#svn/branches/feedback</a></aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-3148.rst">https://github.com/python/peps/blob/main/peps/pep-3148.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-3148.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#naming">Naming</a></li>
|
||
<li><a class="reference internal" href="#interface">Interface</a><ul>
|
||
<li><a class="reference internal" href="#executor">Executor</a></li>
|
||
<li><a class="reference internal" href="#processpoolexecutor">ProcessPoolExecutor</a></li>
|
||
<li><a class="reference internal" href="#threadpoolexecutor">ThreadPoolExecutor</a></li>
|
||
<li><a class="reference internal" href="#future-objects">Future Objects</a><ul>
|
||
<li><a class="reference internal" href="#internal-future-methods">Internal Future Methods</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#module-functions">Module Functions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#check-prime-example">Check Prime Example</a></li>
|
||
<li><a class="reference internal" href="#web-crawl-example">Web Crawl Example</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-3148.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |