mirror of https://github.com/python/peps
272 lines
20 KiB
HTML
272 lines
20 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 3145 – Asynchronous I/O For subprocess.Popen | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-3145/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 3145 – Asynchronous I/O For subprocess.Popen | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-3145/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 3145</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 3145 – Asynchronous I/O For subprocess.Popen</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Eric Pruitt, Charles R. McCreary, Josiah Carlson</dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Removed from consideration by sponsor or authors">Withdrawn</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">04-Aug-2009</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.2</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p></p></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#pep-deferral">PEP Deferral</a></li>
|
||
<li><a class="reference internal" href="#pep-withdrawal">PEP Withdrawal</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>In its present form, the <code class="docutils literal notranslate"><span class="pre">subprocess.Popen</span></code> implementation is prone to
|
||
dead-locking and blocking of the parent Python script while waiting on data
|
||
from the child process. This PEP proposes to make
|
||
<code class="docutils literal notranslate"><span class="pre">subprocess.Popen</span></code> more asynchronous to help alleviate these
|
||
problems.</p>
|
||
</section>
|
||
<section id="pep-deferral">
|
||
<h2><a class="toc-backref" href="#pep-deferral" role="doc-backlink">PEP Deferral</a></h2>
|
||
<p>Further exploration of the concepts covered in this PEP has been deferred
|
||
at least until after <a class="pep reference internal" href="../pep-3156/" title="PEP 3156 – Asynchronous IO Support Rebooted: the “asyncio” Module">PEP 3156</a> has been resolved.</p>
|
||
</section>
|
||
<section id="pep-withdrawal">
|
||
<h2><a class="toc-backref" href="#pep-withdrawal" role="doc-backlink">PEP Withdrawal</a></h2>
|
||
<p>This can be dealt with in the bug tracker. A specific proposal is
|
||
attached to <a class="footnote-reference brackets" href="#id22" id="id1">[11]</a>.</p>
|
||
</section>
|
||
<section id="motivation">
|
||
<h2><a class="toc-backref" href="#motivation" role="doc-backlink">Motivation</a></h2>
|
||
<p>A search for “python asynchronous subprocess” will turn up numerous
|
||
accounts of people wanting to execute a child process and communicate with
|
||
it from time to time reading only the data that is available instead of
|
||
blocking to wait for the program to produce data <a class="footnote-reference brackets" href="#id13" id="id2">[1]</a> <a class="footnote-reference brackets" href="#id14" id="id3">[2]</a> <a class="footnote-reference brackets" href="#id15" id="id4">[3]</a>. The current
|
||
behavior of the <code class="docutils literal notranslate"><span class="pre">subprocess</span></code> module is that when a user sends or receives
|
||
data via the stdin, stderr and stdout file objects, dead locks are common
|
||
and documented <a class="footnote-reference brackets" href="#id16" id="id5">[4]</a> <a class="footnote-reference brackets" href="#id17" id="id6">[5]</a>. While communicate can be used to alleviate some of
|
||
the buffering issues, it will still cause the parent process to block while
|
||
attempting to read data when none is available to be read from the child
|
||
process.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>There is a documented need for asynchronous, non-blocking functionality in
|
||
<code class="docutils literal notranslate"><span class="pre">subprocess.Popen</span></code> <a class="footnote-reference brackets" href="#id18" id="id7">[6]</a> <a class="footnote-reference brackets" href="#id19" id="id8">[7]</a> <a class="footnote-reference brackets" href="#id14" id="id9">[2]</a> <a class="footnote-reference brackets" href="#id15" id="id10">[3]</a>. Inclusion of the code would improve the
|
||
utility of the Python standard library that can be used on Unix based and
|
||
Windows builds of Python. Practically every I/O object in Python has a
|
||
file-like wrapper of some sort. Sockets already act as such and for
|
||
strings there is <code class="docutils literal notranslate"><span class="pre">StringIO</span></code>. Popen can be made to act like a file by simply
|
||
using the methods attached to the <code class="docutils literal notranslate"><span class="pre">subprocess.Popen.stderr</span></code>, stdout and
|
||
stdin file-like objects. But when using the read and write methods of
|
||
those options, you do not have the benefit of asynchronous I/O. In the
|
||
proposed solution the wrapper wraps the asynchronous methods to mimic a
|
||
file object.</p>
|
||
</section>
|
||
<section id="reference-implementation">
|
||
<h2><a class="toc-backref" href="#reference-implementation" role="doc-backlink">Reference Implementation</a></h2>
|
||
<p>I have been maintaining a Google Code repository that contains all of my
|
||
changes including tests and documentation <a class="footnote-reference brackets" href="#id20" id="id11">[9]</a> as well as blog detailing
|
||
the problems I have come across in the development process <a class="footnote-reference brackets" href="#id21" id="id12">[10]</a>.</p>
|
||
<p>I have been working on implementing non-blocking asynchronous I/O in the
|
||
<code class="docutils literal notranslate"><span class="pre">subprocess</span></code> module as well as a wrapper class for <code class="docutils literal notranslate"><span class="pre">subprocess.Popen</span></code>
|
||
that makes it so that an executed process can take the place of a file by
|
||
duplicating all of the methods and attributes that file objects have.</p>
|
||
<p>There are two base functions that have been added to the <code class="docutils literal notranslate"><span class="pre">subprocess.Popen</span></code>
|
||
class: <code class="docutils literal notranslate"><span class="pre">Popen.send</span></code> and <code class="docutils literal notranslate"><span class="pre">Popen._recv</span></code>, each with two separate implementations,
|
||
one for Windows and one for Unix-based systems. The Windows
|
||
implementation uses ctypes to access the functions needed to control pipes
|
||
in the kernel 32 DLL in an asynchronous manner. On Unix based systems,
|
||
the Python interface for file control serves the same purpose. The
|
||
different implementations of <code class="docutils literal notranslate"><span class="pre">Popen.send</span></code> and <code class="docutils literal notranslate"><span class="pre">Popen._recv</span></code> have identical
|
||
arguments to make code that uses these functions work across multiple
|
||
platforms.</p>
|
||
<p>When calling the <code class="docutils literal notranslate"><span class="pre">Popen._recv</span></code> function, it requires the pipe name be
|
||
passed as an argument so there exists the <code class="docutils literal notranslate"><span class="pre">Popen.recv</span></code> function that passes
|
||
selects stdout as the pipe for <code class="docutils literal notranslate"><span class="pre">Popen._recv</span></code> by default. <code class="docutils literal notranslate"><span class="pre">Popen.recv_err</span></code>
|
||
selects stderr as the pipe by default. <code class="docutils literal notranslate"><span class="pre">Popen.recv</span></code> and <code class="docutils literal notranslate"><span class="pre">Popen.recv_err</span></code>
|
||
are much easier to read and understand than <code class="docutils literal notranslate"><span class="pre">Popen._recv('stdout'</span> <span class="pre">...)</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">Popen._recv('stderr'</span> <span class="pre">...)</span></code> respectively.</p>
|
||
<p>Since the <code class="docutils literal notranslate"><span class="pre">Popen._recv</span></code> function does not wait on data to be produced
|
||
before returning a value, it may return empty bytes. <code class="docutils literal notranslate"><span class="pre">Popen.asyncread</span></code>
|
||
handles this issue by returning all data read over a given time
|
||
interval.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">ProcessIOWrapper</span></code> class uses the <code class="docutils literal notranslate"><span class="pre">asyncread</span></code> and <code class="docutils literal notranslate"><span class="pre">asyncwrite</span></code> functions to
|
||
allow a process to act like a file so that there are no blocking issues
|
||
that can arise from using the stdout and stdin file objects produced from
|
||
a <code class="docutils literal notranslate"><span class="pre">subprocess.Popen</span></code> call.</p>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id13" role="doc-footnote">
|
||
<dt class="label" id="id13">[<a href="#id2">1</a>]</dt>
|
||
<dd>[ python-Feature Requests-1191964 ] asynchronous Subprocess
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-bugs-list/2006-December/036524.html">https://mail.python.org/pipermail/python-bugs-list/2006-December/036524.html</a></aside>
|
||
<aside class="footnote brackets" id="id14" role="doc-footnote">
|
||
<dt class="label" id="id14">[2]<em> (<a href='#id3'>1</a>, <a href='#id9'>2</a>) </em></dt>
|
||
<dd>Daily Life in an Ivory Basement : /feb-07/problems-with-subprocess
|
||
<a class="reference external" href="http://ivory.idyll.org/blog/problems-with-subprocess.html">http://ivory.idyll.org/blog/problems-with-subprocess.html</a></aside>
|
||
<aside class="footnote brackets" id="id15" role="doc-footnote">
|
||
<dt class="label" id="id15">[3]<em> (<a href='#id4'>1</a>, <a href='#id10'>2</a>) </em></dt>
|
||
<dd>How can I run an external command asynchronously from Python? - Stack
|
||
Overflow
|
||
<a class="reference external" href="https://stackoverflow.com/q/636561">https://stackoverflow.com/q/636561</a></aside>
|
||
<aside class="footnote brackets" id="id16" role="doc-footnote">
|
||
<dt class="label" id="id16">[<a href="#id5">4</a>]</dt>
|
||
<dd>18.1. subprocess - Subprocess management - Python v2.6.2 documentation
|
||
<a class="reference external" href="https://docs.python.org/2.6/library/subprocess.html#subprocess.Popen.wait">https://docs.python.org/2.6/library/subprocess.html#subprocess.Popen.wait</a></aside>
|
||
<aside class="footnote brackets" id="id17" role="doc-footnote">
|
||
<dt class="label" id="id17">[<a href="#id6">5</a>]</dt>
|
||
<dd>18.1. subprocess - Subprocess management - Python v2.6.2 documentation
|
||
<a class="reference external" href="https://docs.python.org/2.6/library/subprocess.html#subprocess.Popen.kill">https://docs.python.org/2.6/library/subprocess.html#subprocess.Popen.kill</a></aside>
|
||
<aside class="footnote brackets" id="id18" role="doc-footnote">
|
||
<dt class="label" id="id18">[<a href="#id7">6</a>]</dt>
|
||
<dd>Issue 1191964: asynchronous Subprocess - Python tracker
|
||
<a class="reference external" href="https://github.com/python/cpython/issues/41922">https://github.com/python/cpython/issues/41922</a></aside>
|
||
<aside class="footnote brackets" id="id19" role="doc-footnote">
|
||
<dt class="label" id="id19">[<a href="#id8">7</a>]</dt>
|
||
<dd>Module to allow Asynchronous subprocess use on Windows and Posix
|
||
platforms - ActiveState Code
|
||
<a class="reference external" href="https://code.activestate.com/recipes/440554/">https://code.activestate.com/recipes/440554/</a></aside>
|
||
</aside>
|
||
<p>[8] subprocess.rst - subprocdev - Project Hosting on Google Code
|
||
<a class="reference external" href="https://web.archive.org/web/20130306074135/http://code.google.com/p/subprocdev/source/browse/doc/subprocess.rst?spec=svn2c925e935cad0166d5da85e37c742d8e7f609de5&r=2c925e935cad0166d5da85e37c742d8e7f609de5">https://web.archive.org/web/20130306074135/http://code.google.com/p/subprocdev/source/browse/doc/subprocess.rst?spec=svn2c925e935cad0166d5da85e37c742d8e7f609de5&r=2c925e935cad0166d5da85e37c742d8e7f609de5</a></p>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id20" role="doc-footnote">
|
||
<dt class="label" id="id20">[<a href="#id11">9</a>]</dt>
|
||
<dd>subprocdev - Project Hosting on Google Code
|
||
<a class="reference external" href="https://code.google.com/archive/p/subprocdev/">https://code.google.com/archive/p/subprocdev/</a></aside>
|
||
<aside class="footnote brackets" id="id21" role="doc-footnote">
|
||
<dt class="label" id="id21">[<a href="#id12">10</a>]</dt>
|
||
<dd>Python Subprocess Dev
|
||
<a class="reference external" href="https://subdev.blogspot.com/">https://subdev.blogspot.com/</a></aside>
|
||
<aside class="footnote brackets" id="id22" role="doc-footnote">
|
||
<dt class="label" id="id22">[<a href="#id1">11</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/python/cpython/issues/63023">https://github.com/python/cpython/issues/63023</a> – Idle: use pipes instead of
|
||
sockets to talk with user subprocess</aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This P.E.P. is licensed under the Open Publication License;
|
||
<a class="reference external" href="http://www.opencontent.org/openpub/">http://www.opencontent.org/openpub/</a>.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-3145.rst">https://github.com/python/peps/blob/main/peps/pep-3145.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-3145.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#pep-deferral">PEP Deferral</a></li>
|
||
<li><a class="reference internal" href="#pep-withdrawal">PEP Withdrawal</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-3145.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |