mirror of https://github.com/python/peps
276 lines
19 KiB
HTML
276 lines
19 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 399 – Pure Python/C Accelerator Module Compatibility Requirements | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0399/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 399 – Pure Python/C Accelerator Module Compatibility Requirements | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0399/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 399</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 399 – Pure Python/C Accelerator Module Compatibility Requirements</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Brett Cannon <brett at python.org></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Accepted and implementation complete, or no longer active">Final</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Non-normative PEP containing background, guidelines or other information relevant to the Python ecosystem">Informational</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">04-Apr-2011</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.3</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">04-Apr-2011, 12-Apr-2011, 17-Jul-2011, 15-Aug-2011, 01-Jan-2013</dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#details">Details</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>The Python standard library under CPython contains various instances
|
||
of modules implemented in both pure Python and C (either entirely or
|
||
partially). This PEP requires that in these instances that the
|
||
C code <strong>must</strong> pass the test suite used for the pure Python code
|
||
so as to act as much as a drop-in replacement as reasonably possible
|
||
(C- and VM-specific tests are exempt). It is also required that new
|
||
C-based modules lacking a pure Python equivalent implementation get
|
||
special permission to be added to the standard library.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Python has grown beyond the CPython virtual machine (VM). <a class="reference external" href="http://ironpython.net/">IronPython</a>,
|
||
<a class="reference external" href="http://www.jython.org/">Jython</a>, and <a class="reference external" href="http://pypy.org/">PyPy</a> are all currently viable alternatives to the
|
||
CPython VM. The VM ecosystem that has sprung up around the Python
|
||
programming language has led to Python being used in many different
|
||
areas where CPython cannot be used, e.g., Jython allowing Python to be
|
||
used in Java applications.</p>
|
||
<p>A problem all of the VMs other than CPython face is handling modules
|
||
from the standard library that are implemented (to some extent) in C.
|
||
Since other VMs do not typically support the entire <a class="reference external" href="http://docs.python.org/py3k/c-api/index.html">C API of CPython</a>
|
||
they are unable to use the code used to create the module. Oftentimes
|
||
this leads these other VMs to either re-implement the modules in pure
|
||
Python or in the programming language used to implement the VM itself
|
||
(e.g., in C# for IronPython). This duplication of effort between
|
||
CPython, PyPy, Jython, and IronPython is extremely unfortunate as
|
||
implementing a module <strong>at least</strong> in pure Python would help mitigate
|
||
this duplicate effort.</p>
|
||
<p>The purpose of this PEP is to minimize this duplicate effort by
|
||
mandating that all new modules added to Python’s standard library
|
||
<strong>must</strong> have a pure Python implementation <em>unless</em> special dispensation
|
||
is given. This makes sure that a module in the stdlib is available to
|
||
all VMs and not just to CPython (pre-existing modules that do not meet
|
||
this requirement are exempt, although there is nothing preventing
|
||
someone from adding in a pure Python implementation retroactively).</p>
|
||
<p>Re-implementing parts (or all) of a module in C (in the case
|
||
of CPython) is still allowed for performance reasons, but any such
|
||
accelerated code must pass the same test suite (sans VM- or C-specific
|
||
tests) to verify semantics and prevent divergence. To accomplish this,
|
||
the test suite for the module must have comprehensive coverage of the
|
||
pure Python implementation before the acceleration code may be added.</p>
|
||
</section>
|
||
<section id="details">
|
||
<h2><a class="toc-backref" href="#details" role="doc-backlink">Details</a></h2>
|
||
<p>Starting in Python 3.3, any modules added to the standard library must
|
||
have a pure Python implementation. This rule can only be ignored if
|
||
the Python development team grants a special exemption for the module.
|
||
Typically the exemption will be granted only when a module wraps a
|
||
specific C-based library (e.g., <a class="reference external" href="http://docs.python.org/py3k/library/sqlite3.html">sqlite3</a>). In granting an exemption it
|
||
will be recognized that the module will be considered exclusive to
|
||
CPython and not part of Python’s standard library that other VMs are
|
||
expected to support. Usage of <code class="docutils literal notranslate"><span class="pre">ctypes</span></code> to provide an
|
||
API for a C library will continue to be frowned upon as <code class="docutils literal notranslate"><span class="pre">ctypes</span></code>
|
||
lacks compiler guarantees that C code typically relies upon to prevent
|
||
certain errors from occurring (e.g., API changes).</p>
|
||
<p>Even though a pure Python implementation is mandated by this PEP, it
|
||
does not preclude the use of a companion acceleration module. If an
|
||
acceleration module is provided it is to be named the same as the
|
||
module it is accelerating with an underscore attached as a prefix,
|
||
e.g., <code class="docutils literal notranslate"><span class="pre">_warnings</span></code> for <code class="docutils literal notranslate"><span class="pre">warnings</span></code>. The common pattern to access
|
||
the accelerated code from the pure Python implementation is to import
|
||
it with an <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">*</span></code>, e.g., <code class="docutils literal notranslate"><span class="pre">from</span> <span class="pre">_warnings</span> <span class="pre">import</span> <span class="pre">*</span></code>. This is
|
||
typically done at the end of the module to allow it to overwrite
|
||
specific Python objects with their accelerated equivalents. This kind
|
||
of import can also be done before the end of the module when needed,
|
||
e.g., an accelerated base class is provided but is then subclassed by
|
||
Python code. This PEP does not mandate that pre-existing modules in
|
||
the stdlib that lack a pure Python equivalent gain such a module. But
|
||
if people do volunteer to provide and maintain a pure Python
|
||
equivalent (e.g., the PyPy team volunteering their pure Python
|
||
implementation of the <code class="docutils literal notranslate"><span class="pre">csv</span></code> module and maintaining it) then such
|
||
code will be accepted. In those instances the C version is considered
|
||
the reference implementation in terms of expected semantics.</p>
|
||
<p>Any new accelerated code must act as a drop-in replacement as close
|
||
to the pure Python implementation as reasonable. Technical details of
|
||
the VM providing the accelerated code are allowed to differ as
|
||
necessary, e.g., a class being a <code class="docutils literal notranslate"><span class="pre">type</span></code> when implemented in C. To
|
||
verify that the Python and equivalent C code operate as similarly as
|
||
possible, both code bases must be tested using the same tests which
|
||
apply to the pure Python code (tests specific to the C code or any VM
|
||
do not follow under this requirement). The test suite is expected to
|
||
be extensive in order to verify expected semantics.</p>
|
||
<p>Acting as a drop-in replacement also dictates that no public API be
|
||
provided in accelerated code that does not exist in the pure Python
|
||
code. Without this requirement people could accidentally come to rely
|
||
on a detail in the accelerated code which is not made available to
|
||
other VMs that use the pure Python implementation. To help verify
|
||
that the contract of semantic equivalence is being met, a module must
|
||
be tested both with and without its accelerated code as thoroughly as
|
||
possible.</p>
|
||
<p>As an example, to write tests which exercise both the pure Python and
|
||
C accelerated versions of a module, a basic idiom can be followed:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">test.support</span> <span class="kn">import</span> <span class="n">import_fresh_module</span>
|
||
<span class="kn">import</span> <span class="nn">unittest</span>
|
||
|
||
<span class="n">c_heapq</span> <span class="o">=</span> <span class="n">import_fresh_module</span><span class="p">(</span><span class="s1">'heapq'</span><span class="p">,</span> <span class="n">fresh</span><span class="o">=</span><span class="p">[</span><span class="s1">'_heapq'</span><span class="p">])</span>
|
||
<span class="n">py_heapq</span> <span class="o">=</span> <span class="n">import_fresh_module</span><span class="p">(</span><span class="s1">'heapq'</span><span class="p">,</span> <span class="n">blocked</span><span class="o">=</span><span class="p">[</span><span class="s1">'_heapq'</span><span class="p">])</span>
|
||
|
||
|
||
<span class="k">class</span> <span class="nc">ExampleTest</span><span class="p">:</span>
|
||
|
||
<span class="k">def</span> <span class="nf">test_example</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">assertTrue</span><span class="p">(</span><span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">,</span> <span class="s1">'heapify'</span><span class="p">))</span>
|
||
|
||
|
||
<span class="k">class</span> <span class="nc">PyExampleTest</span><span class="p">(</span><span class="n">ExampleTest</span><span class="p">,</span> <span class="n">unittest</span><span class="o">.</span><span class="n">TestCase</span><span class="p">):</span>
|
||
<span class="n">module</span> <span class="o">=</span> <span class="n">py_heapq</span>
|
||
|
||
|
||
<span class="nd">@unittest</span><span class="o">.</span><span class="n">skipUnless</span><span class="p">(</span><span class="n">c_heapq</span><span class="p">,</span> <span class="s1">'requires the C _heapq module'</span><span class="p">)</span>
|
||
<span class="k">class</span> <span class="nc">CExampleTest</span><span class="p">(</span><span class="n">ExampleTest</span><span class="p">,</span> <span class="n">unittest</span><span class="o">.</span><span class="n">TestCase</span><span class="p">):</span>
|
||
<span class="n">module</span> <span class="o">=</span> <span class="n">c_heapq</span>
|
||
|
||
|
||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||
<span class="n">unittest</span><span class="o">.</span><span class="n">main</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The test module defines a base class (<code class="docutils literal notranslate"><span class="pre">ExampleTest</span></code>) with test methods
|
||
that access the <code class="docutils literal notranslate"><span class="pre">heapq</span></code> module through a <code class="docutils literal notranslate"><span class="pre">self.heapq</span></code> class attribute,
|
||
and two subclasses that set this attribute to either the Python or the C
|
||
version of the module. Note that only the two subclasses inherit from
|
||
<code class="docutils literal notranslate"><span class="pre">unittest.TestCase</span></code> – this prevents the <code class="docutils literal notranslate"><span class="pre">ExampleTest</span></code> class from
|
||
being detected as a <code class="docutils literal notranslate"><span class="pre">TestCase</span></code> subclass by <code class="docutils literal notranslate"><span class="pre">unittest</span></code> test discovery.
|
||
A <code class="docutils literal notranslate"><span class="pre">skipUnless</span></code> decorator can be added to the class that tests the C code
|
||
in order to have these tests skipped when the C module is not available.</p>
|
||
<p>If this test were to provide extensive coverage for
|
||
<code class="docutils literal notranslate"><span class="pre">heapq.heappop()</span></code> in the pure Python implementation then the
|
||
accelerated C code would be allowed to be added to CPython’s standard
|
||
library. If it did not, then the test suite would need to be updated
|
||
until proper coverage was provided before the accelerated C code
|
||
could be added.</p>
|
||
<p>To also help with compatibility, C code should use abstract APIs on
|
||
objects to prevent accidental dependence on specific types. For
|
||
instance, if a function accepts a sequence then the C code should
|
||
default to using <code class="docutils literal notranslate"><span class="pre">PyObject_GetItem()</span></code> instead of something like
|
||
<code class="docutils literal notranslate"><span class="pre">PyList_GetItem()</span></code>. C code is allowed to have a fast path if the
|
||
proper <code class="docutils literal notranslate"><span class="pre">PyList_CheckExact()</span></code> is used, but otherwise APIs should work
|
||
with any object that duck types to the proper interface instead of a
|
||
specific type.</p>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0399.rst">https://github.com/python/peps/blob/main/peps/pep-0399.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0399.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#details">Details</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0399.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |