mirror of https://github.com/python/peps
324 lines
22 KiB
HTML
324 lines
22 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 298 – The Locked Buffer Interface | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0298/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 298 – The Locked Buffer Interface | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0298/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 298</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 298 – The Locked Buffer Interface</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Thomas Heller <theller at python.net></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Removed from consideration by sponsor or authors">Withdrawn</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">26-Jul-2002</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">2.3</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">30-Jul-2002, 01-Aug-2002</dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a></li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a></li>
|
||
<li><a class="reference internal" href="#backward-compatibility">Backward Compatibility</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#additional-notes-comments">Additional Notes/Comments</a></li>
|
||
<li><a class="reference internal" href="#community-feedback">Community Feedback</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP proposes an extension to the buffer interface called the
|
||
‘locked buffer interface’.</p>
|
||
<p>The locked buffer interface avoids the flaws of the ‘old’ buffer
|
||
interface <a class="footnote-reference brackets" href="#id2" id="id1">[1]</a> as defined in Python versions up to and including
|
||
2.2, and has the following semantics:</p>
|
||
<ul class="simple">
|
||
<li>The lifetime of the retrieved pointer is clearly defined and
|
||
controlled by the client.</li>
|
||
<li>The buffer size is returned as a ‘size_t’ data type, which
|
||
allows access to large buffers on platforms where <code class="docutils literal notranslate"><span class="pre">sizeof(int)</span>
|
||
<span class="pre">!=</span> <span class="pre">sizeof(void</span> <span class="pre">*)</span></code>.</li>
|
||
</ul>
|
||
<p>(Guido comments: This second sounds like a change we could also
|
||
make to the “old” buffer interface, if we introduce another flag
|
||
bit that’s <em>not</em> part of the default flags.)</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<p>The locked buffer interface exposes new functions which return the
|
||
size and the pointer to the internal memory block of any python
|
||
object which chooses to implement this interface.</p>
|
||
<p>Retrieving a buffer from an object puts this object in a locked
|
||
state during which the buffer may not be freed, resized, or
|
||
reallocated.</p>
|
||
<p>The object must be unlocked again by releasing the buffer if it’s
|
||
no longer used by calling another function in the locked buffer
|
||
interface. If the object never resizes or reallocates the buffer
|
||
during its lifetime, this function may be NULL. Failure to call
|
||
this function (if it is != NULL) is a programming error and may
|
||
have unexpected results.</p>
|
||
<p>The locked buffer interface omits the memory segment model which
|
||
is present in the old buffer interface - only a single memory
|
||
block can be exposed.</p>
|
||
<p>The memory blocks can be accessed without holding the global
|
||
interpreter lock.</p>
|
||
</section>
|
||
<section id="implementation">
|
||
<h2><a class="toc-backref" href="#implementation" role="doc-backlink">Implementation</a></h2>
|
||
<p>Define a new flag in Include/object.h:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">/*</span> <span class="n">PyBufferProcs</span> <span class="n">contains</span> <span class="n">bf_acquirelockedreadbuffer</span><span class="p">,</span>
|
||
<span class="n">bf_acquirelockedwritebuffer</span><span class="p">,</span> <span class="ow">and</span> <span class="n">bf_releaselockedbuffer</span> <span class="o">*/</span>
|
||
<span class="c1">#define Py_TPFLAGS_HAVE_LOCKEDBUFFER (1L<<15)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This flag would be included in <code class="docutils literal notranslate"><span class="pre">Py_TPFLAGS_DEFAULT</span></code>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1">#define Py_TPFLAGS_DEFAULT ( \</span>
|
||
<span class="o">....</span>
|
||
<span class="n">Py_TPFLAGS_HAVE_LOCKEDBUFFER</span> <span class="o">|</span> \
|
||
<span class="o">....</span>
|
||
<span class="mi">0</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Extend the <code class="docutils literal notranslate"><span class="pre">PyBufferProcs</span></code> structure by new fields in
|
||
Include/object.h:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">typedef</span> <span class="n">size_t</span> <span class="p">(</span><span class="o">*</span><span class="n">acquirelockedreadbufferproc</span><span class="p">)(</span><span class="n">PyObject</span> <span class="o">*</span><span class="p">,</span>
|
||
<span class="n">const</span> <span class="n">void</span> <span class="o">**</span><span class="p">);</span>
|
||
<span class="n">typedef</span> <span class="n">size_t</span> <span class="p">(</span><span class="o">*</span><span class="n">acquirelockedwritebufferproc</span><span class="p">)(</span><span class="n">PyObject</span> <span class="o">*</span><span class="p">,</span>
|
||
<span class="n">void</span> <span class="o">**</span><span class="p">);</span>
|
||
<span class="n">typedef</span> <span class="n">void</span> <span class="p">(</span><span class="o">*</span><span class="n">releaselockedbufferproc</span><span class="p">)(</span><span class="n">PyObject</span> <span class="o">*</span><span class="p">);</span>
|
||
|
||
<span class="n">typedef</span> <span class="n">struct</span> <span class="p">{</span>
|
||
<span class="n">getreadbufferproc</span> <span class="n">bf_getreadbuffer</span><span class="p">;</span>
|
||
<span class="n">getwritebufferproc</span> <span class="n">bf_getwritebuffer</span><span class="p">;</span>
|
||
<span class="n">getsegcountproc</span> <span class="n">bf_getsegcount</span><span class="p">;</span>
|
||
<span class="n">getcharbufferproc</span> <span class="n">bf_getcharbuffer</span><span class="p">;</span>
|
||
<span class="o">/*</span> <span class="n">locked</span> <span class="n">buffer</span> <span class="n">interface</span> <span class="n">functions</span> <span class="o">*/</span>
|
||
<span class="n">acquirelockedreadbufferproc</span> <span class="n">bf_acquirelockedreadbuffer</span><span class="p">;</span>
|
||
<span class="n">acquirelockedwritebufferproc</span> <span class="n">bf_acquirelockedwritebuffer</span><span class="p">;</span>
|
||
<span class="n">releaselockedbufferproc</span> <span class="n">bf_releaselockedbuffer</span><span class="p">;</span>
|
||
<span class="p">}</span> <span class="n">PyBufferProcs</span><span class="p">;</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The new fields are present if the <code class="docutils literal notranslate"><span class="pre">Py_TPFLAGS_HAVE_LOCKEDBUFFER</span></code>
|
||
flag is set in the object’s type.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">Py_TPFLAGS_HAVE_LOCKEDBUFFER</span></code> flag implies the
|
||
<code class="docutils literal notranslate"><span class="pre">Py_TPFLAGS_HAVE_GETCHARBUFFER</span></code> flag.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">acquirelockedreadbufferproc</span></code> and <code class="docutils literal notranslate"><span class="pre">acquirelockedwritebufferproc</span></code>
|
||
functions return the size in bytes of the memory block on success,
|
||
and fill in the passed void * pointer on success. If these
|
||
functions fail - either because an error occurs or no memory block
|
||
is exposed - they must set the void * pointer to NULL and raise an
|
||
exception. The return value is undefined in these cases and
|
||
should not be used.</p>
|
||
<p>If calls to these functions succeed, eventually the buffer must be
|
||
released by a call to the <code class="docutils literal notranslate"><span class="pre">releaselockedbufferproc</span></code>, supplying the
|
||
original object as argument. The <code class="docutils literal notranslate"><span class="pre">releaselockedbufferproc</span></code> cannot
|
||
fail. For objects that actually maintain an internal lock count
|
||
it would be a fatal error if the <code class="docutils literal notranslate"><span class="pre">releaselockedbufferproc</span></code> function
|
||
would be called too often, leading to a negative lock count.</p>
|
||
<p>Similar to the ‘old’ buffer interface, any of these functions may
|
||
be set to NULL, but it is strongly recommended to implement the
|
||
<code class="docutils literal notranslate"><span class="pre">releaselockedbufferproc</span></code> function (even if it does nothing) if any
|
||
of the <code class="docutils literal notranslate"><span class="pre">acquireread</span></code>/<code class="docutils literal notranslate"><span class="pre">writelockedbufferproc</span></code> functions are
|
||
implemented, to discourage extension writers from checking for a
|
||
NULL value and not calling it.</p>
|
||
<p>These functions aren’t supposed to be called directly, they are
|
||
called through convenience functions declared in
|
||
Include/abstract.h:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="n">PyObject_AcquireLockedReadBuffer</span><span class="p">(</span><span class="n">PyObject</span> <span class="o">*</span><span class="n">obj</span><span class="p">,</span>
|
||
<span class="n">const</span> <span class="n">void</span> <span class="o">**</span><span class="n">buffer</span><span class="p">,</span>
|
||
<span class="n">size_t</span> <span class="o">*</span><span class="n">buffer_len</span><span class="p">);</span>
|
||
|
||
<span class="nb">int</span> <span class="n">PyObject_AcquireLockedWriteBuffer</span><span class="p">(</span><span class="n">PyObject</span> <span class="o">*</span><span class="n">obj</span><span class="p">,</span>
|
||
<span class="n">void</span> <span class="o">**</span><span class="n">buffer</span><span class="p">,</span>
|
||
<span class="n">size_t</span> <span class="o">*</span><span class="n">buffer_len</span><span class="p">);</span>
|
||
|
||
<span class="n">void</span> <span class="n">PyObject_ReleaseLockedBuffer</span><span class="p">(</span><span class="n">PyObject</span> <span class="o">*</span><span class="n">obj</span><span class="p">);</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The former two functions return 0 on success, set buffer to the
|
||
memory location and buffer_len to the length of the memory block
|
||
in bytes. On failure, or if the locked buffer interface is not
|
||
implemented by obj, they return -1 and set an exception.</p>
|
||
<p>The latter function doesn’t return anything, and cannot fail.</p>
|
||
</section>
|
||
<section id="backward-compatibility">
|
||
<h2><a class="toc-backref" href="#backward-compatibility" role="doc-backlink">Backward Compatibility</a></h2>
|
||
<p>The size of the <code class="docutils literal notranslate"><span class="pre">PyBufferProcs</span></code> structure changes if this proposal
|
||
is implemented, but the type’s <code class="docutils literal notranslate"><span class="pre">tp_flags</span></code> slot can be used to
|
||
determine if the additional fields are present.</p>
|
||
</section>
|
||
<section id="reference-implementation">
|
||
<h2><a class="toc-backref" href="#reference-implementation" role="doc-backlink">Reference Implementation</a></h2>
|
||
<p>An implementation has been uploaded to the SourceForge patch
|
||
manager as <a class="reference external" href="https://bugs.python.org/issue652857">https://bugs.python.org/issue652857</a>.</p>
|
||
</section>
|
||
<section id="additional-notes-comments">
|
||
<h2><a class="toc-backref" href="#additional-notes-comments" role="doc-backlink">Additional Notes/Comments</a></h2>
|
||
<p>Python strings, unicode strings, mmap objects, and array objects
|
||
would expose the locked buffer interface.</p>
|
||
<p>mmap and array objects would actually enter a locked state while
|
||
the buffer is active, this is not needed for strings and unicode
|
||
objects. Resizing locked array objects is not allowed and will
|
||
raise an exception. Whether closing a locked mmap object is an
|
||
error or will only be deferred until the lock count reaches zero
|
||
is an implementation detail.</p>
|
||
<p>Guido recommends</p>
|
||
<blockquote>
|
||
<div>But I’m still very concerned that if most built-in types
|
||
(e.g. strings, bytes) don’t implement the release
|
||
functionality, it’s too easy for an extension to seem to work
|
||
while forgetting to release the buffer.<p>I recommend that at least some built-in types implement the
|
||
acquire/release functionality with a counter, and assert that
|
||
the counter is zero when the object is deleted – if the
|
||
assert fails, someone DECREF’ed their reference to the object
|
||
without releasing it. (The rule should be that you must own a
|
||
reference to the object while you’ve acquired the object.)</p>
|
||
<p>For strings that might be impractical because the string
|
||
object would have to grow 4 bytes to hold the counter; but the
|
||
new bytes object (<a class="pep reference internal" href="../pep-0296/" title="PEP 296 – Adding a bytes Object Type">PEP 296</a>) could easily implement the counter,
|
||
and the array object too – that way there will be plenty of
|
||
opportunity to test proper use of the protocol.</p>
|
||
</div></blockquote>
|
||
</section>
|
||
<section id="community-feedback">
|
||
<h2><a class="toc-backref" href="#community-feedback" role="doc-backlink">Community Feedback</a></h2>
|
||
<p>Greg Ewing doubts the locked buffer interface is needed at all, he
|
||
thinks the normal buffer interface could be used if the pointer is
|
||
(re)fetched each time it’s used. This seems to be dangerous,
|
||
because even innocent looking calls to the Python API like
|
||
<code class="docutils literal notranslate"><span class="pre">Py_DECREF()</span></code> may trigger execution of arbitrary Python code.</p>
|
||
<p>The first version of this proposal didn’t have the release
|
||
function, but it turned out that this would have been too
|
||
restrictive: mmap and array objects wouldn’t have been able to
|
||
implement it, because mmap objects can be closed anytime if not
|
||
locked, and array objects could resize or reallocate the buffer.</p>
|
||
<p>This PEP will probably be rejected because nobody except the
|
||
author needs it.</p>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id2" role="doc-footnote">
|
||
<dt class="label" id="id2">[<a href="#id1">1</a>]</dt>
|
||
<dd>The buffer interface
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-dev/2000-October/009974.html">https://mail.python.org/pipermail/python-dev/2000-October/009974.html</a></aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0298.rst">https://github.com/python/peps/blob/main/peps/pep-0298.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0298.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a></li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a></li>
|
||
<li><a class="reference internal" href="#backward-compatibility">Backward Compatibility</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#additional-notes-comments">Additional Notes/Comments</a></li>
|
||
<li><a class="reference internal" href="#community-feedback">Community Feedback</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0298.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |