mirror of https://github.com/python/peps
670 lines
64 KiB
HTML
670 lines
64 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 711 – PyBI: a standard format for distributing Python Binaries | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0711/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 711 – PyBI: a standard format for distributing Python Binaries | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0711/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 711</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 711 – PyBI: a standard format for distributing Python Binaries</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Nathaniel J. Smith <njs at pobox.com></dd>
|
||
<dt class="field-even">PEP-Delegate<span class="colon">:</span></dt>
|
||
<dd class="field-even">TODO</dd>
|
||
<dt class="field-odd">Discussions-To<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://discuss.python.org/t/pep-711-pybi-a-standard-format-for-distributing-python-binaries/25547">Discourse thread</a></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Proposal under active discussion and revision">Draft</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Topic<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="../topic/packaging/">Packaging</a></dd>
|
||
<dt class="field-odd">Created<span class="colon">:</span></dt>
|
||
<dd class="field-odd">06-Apr-2023</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="https://discuss.python.org/t/pep-711-pybi-a-standard-format-for-distributing-python-binaries/25547" title="Discourse thread">06-Apr-2023</a></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#examples">Examples</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#filename">Filename</a></li>
|
||
<li><a class="reference internal" href="#file-contents">File contents</a><ul>
|
||
<li><a class="reference internal" href="#pybi-specific-core-metadata">Pybi-specific core metadata</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#symlinks">Symlinks</a><ul>
|
||
<li><a class="reference internal" href="#representing-symlinks-in-zip-files">Representing symlinks in zip files</a></li>
|
||
<li><a class="reference internal" href="#representing-symlinks-in-record-files">Representing symlinks in RECORD files</a></li>
|
||
<li><a class="reference internal" href="#storing-symlinks-in-pybi-files">Storing symlinks in <code class="docutils literal notranslate"><span class="pre">pybi</span></code> files</a></li>
|
||
<li><a class="reference internal" href="#limitations">Limitations</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#non-normative-comments">Non-normative comments</a><ul>
|
||
<li><a class="reference internal" href="#why-not-just-use-conda">Why not just use conda?</a></li>
|
||
<li><a class="reference internal" href="#sdists-or-not">Sdists (or not)</a></li>
|
||
<li><a class="reference internal" href="#what-packages-should-be-bundled-inside-a-pybi">What packages should be bundled inside a pybi?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#security-implications">Security Implications</a></li>
|
||
<li><a class="reference internal" href="#how-to-teach-this">How to Teach This</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>“Like wheels, but instead of a pre-built python package, it’s a
|
||
pre-built python interpreter”</p>
|
||
</section>
|
||
<section id="motivation">
|
||
<h2><a class="toc-backref" href="#motivation" role="doc-backlink">Motivation</a></h2>
|
||
<p>End goal: Pypi.org has pre-built packages for all Python versions on all
|
||
popular platforms, so automated tools can easily grab any of them and
|
||
set it up. It becomes quick and easy to try Python prereleases, pin
|
||
Python versions in CI, make a temporary environment to reproduce a bug
|
||
report that only happens on a specific Python point release, etc.</p>
|
||
<p>First step (this PEP): define a standard packaging file format to hold pre-built
|
||
Python interpreters, that reuses existing Python packaging standards as much as
|
||
possible.</p>
|
||
</section>
|
||
<section id="examples">
|
||
<h2><a class="toc-backref" href="#examples" role="doc-backlink">Examples</a></h2>
|
||
<p>Example pybi builds are available at <a class="reference external" href="https://pybi.vorpus.org">pybi.vorpus.org</a>. They’re zip files, so you can unpack them and poke
|
||
around inside if you want to get a feel for how they’re laid out.</p>
|
||
<p>You can also look at the <a class="reference external" href="https://github.com/njsmith/pybi-tools">tooling I used to create them</a>.</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<section id="filename">
|
||
<h3><a class="toc-backref" href="#filename" role="doc-backlink">Filename</a></h3>
|
||
<p>Filename: <code class="docutils literal notranslate"><span class="pre">{distribution}-{version}[-{build</span> <span class="pre">tag}]-{platform</span> <span class="pre">tag}.pybi</span></code></p>
|
||
<p>This matches the wheel file format defined in <a class="pep reference internal" href="../pep-0427/" title="PEP 427 – The Wheel Binary Package Format 1.0">PEP 427</a>, except dropping the
|
||
<code class="docutils literal notranslate"><span class="pre">{python</span> <span class="pre">tag}</span></code> and <code class="docutils literal notranslate"><span class="pre">{abi</span> <span class="pre">tag}</span></code> and changing the extension from <code class="docutils literal notranslate"><span class="pre">.whl</span></code> →
|
||
<code class="docutils literal notranslate"><span class="pre">.pybi</span></code>.</p>
|
||
<p>For example:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">cpython-3.9.3-manylinux_2014.pybi</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">cpython-3.10b2-win_amd64.pybi</span></code></li>
|
||
</ul>
|
||
<p>Just like for wheels, if a pybi supports multiple platforms, you can
|
||
separate them by dots to make a “compressed tag set”:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">cpython-3.9.5-macosx_11_0_x86_64.macosx_11_0_arm64.pybi</span></code></li>
|
||
</ul>
|
||
<p>(Though in practice this probably won’t be used much, e.g. the above
|
||
filename is more idiomatically written as
|
||
<code class="docutils literal notranslate"><span class="pre">cpython-3.9.5-macosx_11_0_universal2.pybi</span></code>.)</p>
|
||
</section>
|
||
<section id="file-contents">
|
||
<h3><a class="toc-backref" href="#file-contents" role="doc-backlink">File contents</a></h3>
|
||
<p>A <code class="docutils literal notranslate"><span class="pre">.pybi</span></code> file is a zip file, that can be unpacked directly into an
|
||
arbitrary location and then used as a self-contained Python environment.
|
||
There’s no <code class="docutils literal notranslate"><span class="pre">.data</span></code> directory or install scheme keys, because the
|
||
Python environment knows which install scheme it’s using, so it can just
|
||
put things in the right places to start with.</p>
|
||
<p>The “arbitrary location” part is important: the pybi can’t contain any
|
||
hardcoded absolute paths. In particular, any preinstalled scripts MUST
|
||
NOT embed absolute paths in their shebang lines.</p>
|
||
<p>Similar to wheels’ <code class="docutils literal notranslate"><span class="pre"><package>-<version>.dist-info</span></code> directory, the pybi archive
|
||
must contain a top-level directory named <code class="docutils literal notranslate"><span class="pre">pybi-info/</span></code>. (Rationale: calling it
|
||
<code class="docutils literal notranslate"><span class="pre">pybi-info</span></code> instead <code class="docutils literal notranslate"><span class="pre">dist-info</span></code> makes sure that tools don’t get confused
|
||
about which kind of metadata they’re looking at; leaving off the
|
||
<code class="docutils literal notranslate"><span class="pre">{name}-{version}</span></code> part is fine because only one pybi can be installed into a
|
||
given directory.) The <code class="docutils literal notranslate"><span class="pre">pybi-info/</span></code> directory contains at least the following
|
||
files:</p>
|
||
<ul>
|
||
<li><code class="docutils literal notranslate"><span class="pre">.../PYBI</span></code>: metadata about the archive itself, in the same
|
||
RFC822-ish format as <code class="docutils literal notranslate"><span class="pre">METADATA</span></code> and <code class="docutils literal notranslate"><span class="pre">WHEEL</span></code> files:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Pybi</span><span class="o">-</span><span class="n">Version</span><span class="p">:</span> <span class="mf">1.0</span>
|
||
<span class="n">Generator</span><span class="p">:</span> <span class="p">{</span><span class="n">name</span><span class="p">}</span> <span class="p">{</span><span class="n">version</span><span class="p">}</span>
|
||
<span class="n">Tag</span><span class="p">:</span> <span class="p">{</span><span class="n">platform</span> <span class="n">tag</span><span class="p">}</span>
|
||
<span class="n">Tag</span><span class="p">:</span> <span class="p">{</span><span class="n">another</span> <span class="n">platform</span> <span class="n">tag</span><span class="p">}</span>
|
||
<span class="n">Tag</span><span class="p">:</span> <span class="p">{</span><span class="o">...</span><span class="ow">and</span> <span class="n">so</span> <span class="n">on</span><span class="o">...</span><span class="p">}</span>
|
||
<span class="n">Build</span><span class="p">:</span> <span class="mi">1</span> <span class="c1"># optional</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">.../RECORD</span></code>: same as in wheels, except see the note about
|
||
symlinks, below.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">.../METADATA</span></code>: In the same format as described in the current core
|
||
metadata spec, except that the following keys are forbidden because
|
||
they don’t make sense:<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">Requires-Dist</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Provides-Extra</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Requires-Python</span></code></li>
|
||
</ul>
|
||
<p>And also there are some new, required keys described below.</p>
|
||
</li>
|
||
</ul>
|
||
<section id="pybi-specific-core-metadata">
|
||
<h4><a class="toc-backref" href="#pybi-specific-core-metadata" role="doc-backlink">Pybi-specific core metadata</a></h4>
|
||
<p>Here’s an example of the new <code class="docutils literal notranslate"><span class="pre">METADATA</span></code> fields, before we give the full details:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Pybi</span><span class="o">-</span><span class="n">Environment</span><span class="o">-</span><span class="n">Marker</span><span class="o">-</span><span class="n">Variables</span><span class="p">:</span> <span class="p">{</span><span class="s2">"implementation_name"</span><span class="p">:</span> <span class="s2">"cpython"</span><span class="p">,</span> <span class="s2">"implementation_version"</span><span class="p">:</span> <span class="s2">"3.10.8"</span><span class="p">,</span> <span class="s2">"os_name"</span><span class="p">:</span> <span class="s2">"posix"</span><span class="p">,</span> <span class="s2">"platform_machine"</span><span class="p">:</span> <span class="s2">"x86_64"</span><span class="p">,</span> <span class="s2">"platform_system"</span><span class="p">:</span> <span class="s2">"Linux"</span><span class="p">,</span> <span class="s2">"python_full_version"</span><span class="p">:</span> <span class="s2">"3.10.8"</span><span class="p">,</span> <span class="s2">"platform_python_implementation"</span><span class="p">:</span> <span class="s2">"CPython"</span><span class="p">,</span> <span class="s2">"python_version"</span><span class="p">:</span> <span class="s2">"3.10"</span><span class="p">,</span> <span class="s2">"sys_platform"</span><span class="p">:</span> <span class="s2">"linux"</span><span class="p">}</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Paths</span><span class="p">:</span> <span class="p">{</span><span class="s2">"stdlib"</span><span class="p">:</span> <span class="s2">"lib/python3.10"</span><span class="p">,</span> <span class="s2">"platstdlib"</span><span class="p">:</span> <span class="s2">"lib/python3.10"</span><span class="p">,</span> <span class="s2">"purelib"</span><span class="p">:</span> <span class="s2">"lib/python3.10/site-packages"</span><span class="p">,</span> <span class="s2">"platlib"</span><span class="p">:</span> <span class="s2">"lib/python3.10/site-packages"</span><span class="p">,</span> <span class="s2">"include"</span><span class="p">:</span> <span class="s2">"include/python3.10"</span><span class="p">,</span> <span class="s2">"platinclude"</span><span class="p">:</span> <span class="s2">"include/python3.10"</span><span class="p">,</span> <span class="s2">"scripts"</span><span class="p">:</span> <span class="s2">"bin"</span><span class="p">,</span> <span class="s2">"data"</span><span class="p">:</span> <span class="s2">"."</span><span class="p">}</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp310</span><span class="o">-</span><span class="n">cp310</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp310</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp310</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp39</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp38</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp37</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp36</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp35</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp34</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp33</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">cp32</span><span class="o">-</span><span class="n">abi3</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py310</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py3</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py39</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py38</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py37</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py36</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py35</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py34</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py33</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py32</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py31</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py30</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="n">PLATFORM</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py310</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py3</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py39</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py38</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py37</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py36</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py35</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py34</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py33</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py32</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py31</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
<span class="n">Pybi</span><span class="o">-</span><span class="n">Wheel</span><span class="o">-</span><span class="n">Tag</span><span class="p">:</span> <span class="n">py30</span><span class="o">-</span><span class="n">none</span><span class="o">-</span><span class="nb">any</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Specification:</p>
|
||
<ul>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Pybi-Environment-Marker-Variables</span></code>: The value of all PEP 508
|
||
environment marker variables that are static across installs of this
|
||
Pybi, as a JSON dict. So for example:<ul>
|
||
<li><code class="docutils literal notranslate"><span class="pre">python_version</span></code> will always be present, because a Python 3.10 package
|
||
always has <code class="docutils literal notranslate"><span class="pre">python_version</span> <span class="pre">==</span> <span class="pre">"3.10"</span></code>.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">platform_version</span></code> will generally not be present, because it gives
|
||
detailed information about the OS where Python is running, for example:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1">#60-Ubuntu SMP Thu May 6 07:46:32 UTC 2021</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><code class="docutils literal notranslate"><span class="pre">platform_release</span></code> has similar issues.</p>
|
||
</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">platform_machine</span></code> will <em>usually</em> be present, except for macOS universal2
|
||
pybis: these can potentially be run in either x86-64 or arm64 mode, and we
|
||
don’t know which until the interpreter is actually invoked, so we can’t
|
||
record it in static metadata.</li>
|
||
</ul>
|
||
<p><strong>Rationale:</strong> In many cases, this should allow a resolver running on Linux
|
||
to compute package pins for a Python environment on Windows, or vice-versa,
|
||
so long as the resolver has access to the target platform’s .pybi file. (Note
|
||
that <code class="docutils literal notranslate"><span class="pre">Requires-Python</span></code> constraints can be checked by using the
|
||
<code class="docutils literal notranslate"><span class="pre">python_full_version</span></code> value.) While we have to leave out a few keys
|
||
sometimes, they’re either fairly useless (<code class="docutils literal notranslate"><span class="pre">platform_version</span></code>,
|
||
<code class="docutils literal notranslate"><span class="pre">platform_release</span></code>) or can be reconstructed by the resolver
|
||
(<code class="docutils literal notranslate"><span class="pre">platform_machine</span></code>).</p>
|
||
<p>The markers are also just generally useful information to have
|
||
accessible. For example, if you have a <code class="docutils literal notranslate"><span class="pre">pypy3-7.3.2</span></code> pybi, and you
|
||
want to know what version of the Python language that supports, then
|
||
that’s recorded in the <code class="docutils literal notranslate"><span class="pre">python_version</span></code> marker.</p>
|
||
<p>(Note: we may want to deprecate/remove <code class="docutils literal notranslate"><span class="pre">platform_version</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">platform_release</span></code>? They’re problematic and I can’t figure out any cases
|
||
where they’re useful. But that’s out of scope of this particular PEP.)</p>
|
||
</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Pybi-Paths</span></code>: The install paths needed to install wheels (same keys
|
||
as <code class="docutils literal notranslate"><span class="pre">sysconfig.get_paths()</span></code>), as relative paths starting at the root
|
||
of the zip file, as a JSON dict.<p>These paths MUST be written in Unix format, using forward slashes as
|
||
a separator, not backslashes.</p>
|
||
<p>It must be possible to invoke the Python interpreter by running
|
||
<code class="docutils literal notranslate"><span class="pre">{paths["scripts"]}/python</span></code>. If there are alternative interpreter
|
||
entry points (e.g. <code class="docutils literal notranslate"><span class="pre">pythonw</span></code> for Windows GUI apps), then they
|
||
should also be in that directory under their conventional names, with
|
||
no version number attached. (You can <em>also</em> have a <code class="docutils literal notranslate"><span class="pre">python3.11</span></code>
|
||
symlink if you want; there’s no rule against that. It’s just that
|
||
<code class="docutils literal notranslate"><span class="pre">python</span></code> has to exist and work.)</p>
|
||
<p><strong>Rationale:</strong> <code class="docutils literal notranslate"><span class="pre">Pybi-Paths</span></code> and <code class="docutils literal notranslate"><span class="pre">Pybi-Wheel-Tag</span></code>s (see below) are
|
||
together enough to let an installer choose wheels and install them into an
|
||
unpacked pybi environment, without invoking Python. Besides, we need to write
|
||
down the interpreter location somewhere, so it’s two birds with one stone.</p>
|
||
</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Pybi-Wheel-Tag</span></code>: The wheel tags supported by this interpreter, in
|
||
preference order (most-preferred first, least-preferred last), except
|
||
that the special platform tag <code class="docutils literal notranslate"><span class="pre">PLATFORM</span></code> should replace any
|
||
platform tags that depend on the final installation system.<p><strong>Discussion:</strong> It would be nice™ if installers could compute a pybi’s
|
||
corresponding wheel tags ahead of time, so that they could install
|
||
wheels into the unpacked pybi without needing to actually invoke the
|
||
python interpreter to query its tags – both for efficiency and to
|
||
allow for more exotic use cases like setting up a Windows environment
|
||
from a Linux host.</p>
|
||
<p>But unfortunately, it’s impossible to compute the full set of
|
||
platform tags supported by a Python installation ahead of time,
|
||
because they can depend on the final system:</p>
|
||
<ul class="simple">
|
||
<li>A pybi tagged <code class="docutils literal notranslate"><span class="pre">manylinux_2_12_x86_64</span></code> can always use wheels
|
||
tagged as <code class="docutils literal notranslate"><span class="pre">manylinux_2_12_x86_64</span></code>. It also <em>might</em> be able to
|
||
use wheels tagged <code class="docutils literal notranslate"><span class="pre">manylinux_2_17_x86_64</span></code>, but only if the final
|
||
installation system has glibc 2.17+.</li>
|
||
<li>A pybi tagged <code class="docutils literal notranslate"><span class="pre">macosx_11_0_universal2</span></code> (= x86-64 + arm64 support
|
||
in the same binary) might be able to use wheels tagged as
|
||
<code class="docutils literal notranslate"><span class="pre">macosx_11_0_arm64</span></code>, but only if it’s installed on an “Apple
|
||
Silicon” machine and running in arm64 mode.</li>
|
||
</ul>
|
||
<p>In these two cases, an installation tool can still work out the
|
||
appropriate set of wheel tags by computing the local platform tags,
|
||
taking the wheel tag templates from <code class="docutils literal notranslate"><span class="pre">Pybi-Wheel-Tag</span></code>, and swapping
|
||
in the actual supported platforms in place of the magic <code class="docutils literal notranslate"><span class="pre">PLATFORM</span></code>
|
||
string.</p>
|
||
<p>However, there are other cases that are even more complicated:</p>
|
||
<ul>
|
||
<li><dl>
|
||
<dt>You can (usually) run both 32- and 64-bit apps on 64-bit Windows. So a pybi</dt><dd>installer might compute the set of allowable pybi tags on the current
|
||
platform as [<code class="docutils literal notranslate"><span class="pre">win32</span></code>, <code class="docutils literal notranslate"><span class="pre">win_amd64</span></code>]. But you can’t then just take that
|
||
set and swap it into the pybi’s wheel tag template or you get nonsense:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
|
||
<span class="s2">"cp39-cp39-win32"</span><span class="p">,</span>
|
||
<span class="s2">"cp39-cp39-win_amd64"</span><span class="p">,</span>
|
||
<span class="s2">"cp39-abi3-win32"</span><span class="p">,</span>
|
||
<span class="s2">"cp39-abi3-win_amd64"</span><span class="p">,</span>
|
||
<span class="o">...</span>
|
||
<span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>To handle this, the installer needs to somehow understand that a
|
||
<code class="docutils literal notranslate"><span class="pre">manylinux_2_12_x86_64</span></code> pybi can use a <code class="docutils literal notranslate"><span class="pre">manylinux_2_17_x86_64</span></code> wheel
|
||
as long as those are both valid tags on the current machine, but a
|
||
<code class="docutils literal notranslate"><span class="pre">win32</span></code> pybi <em>can’t</em> use a <code class="docutils literal notranslate"><span class="pre">win_amd64</span></code> wheel, even if those are both
|
||
valid tags on the current machine.</p>
|
||
</dd>
|
||
</dl>
|
||
</li>
|
||
<li>A pybi tagged <code class="docutils literal notranslate"><span class="pre">macosx_11_0_universal2</span></code> might be able to use
|
||
wheels tagged as <code class="docutils literal notranslate"><span class="pre">macosx_11_0_x86_64</span></code>, but only if it’s
|
||
installed on an x86-64 machine <em>or</em> it’s installed on an ARM
|
||
machine <em>and</em> the interpreter is invoked with the magic
|
||
incantation that tells macOS to run a binary in x86-64 mode. So
|
||
how the installer plans to invoke the pybi matters too!</li>
|
||
</ul>
|
||
<p>So actually using <code class="docutils literal notranslate"><span class="pre">Pybi-Wheel-Tag</span></code> values is less trivial than it
|
||
might seem, and they’re probably only useful with fairly
|
||
sophisticated tooling. But, smart pybi installers will already have
|
||
to understand a lot of these platform compatibility issues in order
|
||
to select a working pybi, and for the cross-platform
|
||
pinning/environment building case, users can potentially provide
|
||
whatever information is needed to disambiguate exactly what platform
|
||
they’re targeting. So, it’s still useful enough to include in the PyBI
|
||
metadata – tools that don’t find it useful can simply ignore it.</p>
|
||
</li>
|
||
</ul>
|
||
<p>You can probably generate these metadata values by running this script on the
|
||
built interpreter:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">packaging.markers</span>
|
||
<span class="kn">import</span> <span class="nn">packaging.tags</span>
|
||
<span class="kn">import</span> <span class="nn">sysconfig</span>
|
||
<span class="kn">import</span> <span class="nn">os.path</span>
|
||
<span class="kn">import</span> <span class="nn">json</span>
|
||
<span class="kn">import</span> <span class="nn">sys</span>
|
||
|
||
<span class="n">marker_vars</span> <span class="o">=</span> <span class="n">packaging</span><span class="o">.</span><span class="n">markers</span><span class="o">.</span><span class="n">default_environment</span><span class="p">()</span>
|
||
<span class="c1"># Delete any keys that depend on the final installation</span>
|
||
<span class="k">del</span> <span class="n">marker_vars</span><span class="p">[</span><span class="s2">"platform_release"</span><span class="p">]</span>
|
||
<span class="k">del</span> <span class="n">marker_vars</span><span class="p">[</span><span class="s2">"platform_version"</span><span class="p">]</span>
|
||
<span class="c1"># Darwin binaries are often multi-arch, so play it safe and</span>
|
||
<span class="c1"># delete the architecture marker. (Better would be to only</span>
|
||
<span class="c1"># do this if the pybi actually is multi-arch.)</span>
|
||
<span class="k">if</span> <span class="n">marker_vars</span><span class="p">[</span><span class="s2">"sys_platform"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"darwin"</span><span class="p">:</span>
|
||
<span class="k">del</span> <span class="n">marker_vars</span><span class="p">[</span><span class="s2">"platform_machine"</span><span class="p">]</span>
|
||
|
||
<span class="c1"># Copied and tweaked version of packaging.tags.sys_tags</span>
|
||
<span class="n">tags</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">interp_name</span> <span class="o">=</span> <span class="n">packaging</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">interpreter_name</span><span class="p">()</span>
|
||
<span class="k">if</span> <span class="n">interp_name</span> <span class="o">==</span> <span class="s2">"cp"</span><span class="p">:</span>
|
||
<span class="n">tags</span> <span class="o">+=</span> <span class="nb">list</span><span class="p">(</span><span class="n">packaging</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">cpython_tags</span><span class="p">(</span><span class="n">platforms</span><span class="o">=</span><span class="p">[</span><span class="s2">"xyzzy"</span><span class="p">]))</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">tags</span> <span class="o">+=</span> <span class="nb">list</span><span class="p">(</span><span class="n">packaging</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">generic_tags</span><span class="p">(</span><span class="n">platforms</span><span class="o">=</span><span class="p">[</span><span class="s2">"xyzzy"</span><span class="p">]))</span>
|
||
|
||
<span class="n">tags</span> <span class="o">+=</span> <span class="nb">list</span><span class="p">(</span><span class="n">packaging</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">compatible_tags</span><span class="p">(</span><span class="n">platforms</span><span class="o">=</span><span class="p">[</span><span class="s2">"xyzzy"</span><span class="p">]))</span>
|
||
|
||
<span class="c1"># Gross hack: packaging.tags normalizes platforms by lowercasing them,</span>
|
||
<span class="c1"># so we generate the tags with a unique string and then replace it</span>
|
||
<span class="c1"># with our special uppercase placeholder.</span>
|
||
<span class="n">str_tags</span> <span class="o">=</span> <span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">t</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"xyzzy"</span><span class="p">,</span> <span class="s2">"PLATFORM"</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">tags</span><span class="p">]</span>
|
||
|
||
<span class="p">(</span><span class="n">base_path</span><span class="p">,)</span> <span class="o">=</span> <span class="n">sysconfig</span><span class="o">.</span><span class="n">get_config_vars</span><span class="p">(</span><span class="s2">"installed_base"</span><span class="p">)</span>
|
||
<span class="c1"># For some reason, macOS framework builds report their</span>
|
||
<span class="c1"># installed_base as a directory deep inside the framework.</span>
|
||
<span class="k">while</span> <span class="s2">"Python.framework"</span> <span class="ow">in</span> <span class="n">base_path</span><span class="p">:</span>
|
||
<span class="n">base_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">base_path</span><span class="p">)</span>
|
||
<span class="n">paths</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">base_path</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">"</span><span class="p">,</span> <span class="s2">"/"</span><span class="p">)</span> <span class="k">for</span> <span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span> <span class="ow">in</span> <span class="n">sysconfig</span><span class="o">.</span><span class="n">get_paths</span><span class="p">()</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||
|
||
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">({</span><span class="s2">"marker_vars"</span><span class="p">:</span> <span class="n">marker_vars</span><span class="p">,</span> <span class="s2">"tags"</span><span class="p">:</span> <span class="n">str_tags</span><span class="p">,</span> <span class="s2">"paths"</span><span class="p">:</span> <span class="n">paths</span><span class="p">},</span> <span class="n">sys</span><span class="o">.</span><span class="n">stdout</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This emits a JSON dict on stdout with separate entries for each set of
|
||
pybi-specific tags.</p>
|
||
</section>
|
||
</section>
|
||
<section id="symlinks">
|
||
<h3><a class="toc-backref" href="#symlinks" role="doc-backlink">Symlinks</a></h3>
|
||
<p>Currently, symlinks are used by default in all Unix Python installs (e.g.,
|
||
<code class="docutils literal notranslate"><span class="pre">bin/python3</span> <span class="pre">-></span> <span class="pre">bin/python3.9</span></code>). And furthermore, symlinks are <em>required</em> to
|
||
store macOS framework builds in <code class="docutils literal notranslate"><span class="pre">.pybi</span></code> files. So, unlike wheel files, we
|
||
absolutely have to support symlinks in <code class="docutils literal notranslate"><span class="pre">.pybi</span></code> files for them to be useful at
|
||
all.</p>
|
||
<section id="representing-symlinks-in-zip-files">
|
||
<h4><a class="toc-backref" href="#representing-symlinks-in-zip-files" role="doc-backlink">Representing symlinks in zip files</a></h4>
|
||
<p>The de-facto standard for representing symlinks in zip files is the
|
||
Info-Zip symlink extension, which works as follows:</p>
|
||
<ul class="simple">
|
||
<li>The symlink’s target path is stored as if it were the file contents</li>
|
||
<li>The top 4 bits of the Unix permissions field are set to <code class="docutils literal notranslate"><span class="pre">0xa</span></code>,
|
||
i.e.: <code class="docutils literal notranslate"><span class="pre">permissions</span> <span class="pre">&</span> <span class="pre">0xf000</span> <span class="pre">==</span> <span class="pre">0xa000</span></code></li>
|
||
<li>The Unix permissions field, in turn, is stored as the top 16 bits of
|
||
the “external attributes” field.</li>
|
||
</ul>
|
||
<p>So if using Python’s <code class="docutils literal notranslate"><span class="pre">zipfile</span></code> module, you can check whether a
|
||
<code class="docutils literal notranslate"><span class="pre">ZipInfo</span></code> represents a symlink by doing:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="p">(</span><span class="n">zip_info</span><span class="o">.</span><span class="n">external_attr</span> <span class="o">>></span> <span class="mi">16</span><span class="p">)</span> <span class="o">&</span> <span class="mh">0xf000</span> <span class="o">==</span> <span class="mh">0xa000</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Or if using Rust’s <code class="docutils literal notranslate"><span class="pre">zip</span></code> crate, the equivalent check is:</p>
|
||
<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">fn</span> <span class="nf">is_symlink</span><span class="p">(</span><span class="n">zip_file</span>: <span class="kp">&</span><span class="nc">zip</span>::<span class="n">ZipFile</span><span class="p">)</span><span class="w"> </span>-> <span class="kt">bool</span> <span class="p">{</span>
|
||
<span class="w"> </span><span class="k">match</span><span class="w"> </span><span class="n">zip_file</span><span class="p">.</span><span class="n">unix_mode</span><span class="p">()</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="nb">Some</span><span class="p">(</span><span class="n">mode</span><span class="p">)</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="n">mode</span><span class="w"> </span><span class="o">&</span><span class="w"> </span><span class="mh">0xf000</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="mh">0xa000</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="nb">None</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="kc">false</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="p">}</span>
|
||
<span class="p">}</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>If you’re on Unix, your <code class="docutils literal notranslate"><span class="pre">zip</span></code> and <code class="docutils literal notranslate"><span class="pre">unzip</span></code> commands probably understands this
|
||
format already.</p>
|
||
</section>
|
||
<section id="representing-symlinks-in-record-files">
|
||
<h4><a class="toc-backref" href="#representing-symlinks-in-record-files" role="doc-backlink">Representing symlinks in RECORD files</a></h4>
|
||
<p>Normally, a <code class="docutils literal notranslate"><span class="pre">RECORD</span></code> file lists each file + its hash + its length:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>my/favorite/file,sha256=...,12345
|
||
</pre></div>
|
||
</div>
|
||
<p>For symlinks, we instead write:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>name/of/symlink,symlink=path/to/symlink/target,
|
||
</pre></div>
|
||
</div>
|
||
<p>That is: we use a special “hash function” called <code class="docutils literal notranslate"><span class="pre">symlink</span></code>, and then
|
||
store the actual symlink target as the “hash value”. And the length is
|
||
left empty.</p>
|
||
<p><strong>Rationale:</strong> we’re already committed to the <code class="docutils literal notranslate"><span class="pre">RECORD</span></code> file containing a
|
||
redundant check on everything in the main archive, so for symlinks we at least
|
||
need to store some kind of hash, plus some kind of flag to indicate that this is
|
||
a symlink. Given that symlink target strings are roughly the same size as a
|
||
hash, we might as well store them directly. This also makes the symlink
|
||
information easier to access for tools that don’t understand the Info-Zip
|
||
symlink extension, and makes it possible to losslessly unpack and repack a Unix
|
||
pybi on a Windows system, which someone might find handy at some point.</p>
|
||
</section>
|
||
<section id="storing-symlinks-in-pybi-files">
|
||
<h4><a class="toc-backref" href="#storing-symlinks-in-pybi-files" role="doc-backlink">Storing symlinks in <code class="docutils literal notranslate"><span class="pre">pybi</span></code> files</a></h4>
|
||
<p>When a pybi creator stores a symlink, they MUST use both of the
|
||
mechanisms defined above: storing it in the zip archive directly using
|
||
the Info-Zip representation, and also recording it in the <code class="docutils literal notranslate"><span class="pre">RECORD</span></code>
|
||
file.</p>
|
||
<p>Pybi consumers SHOULD validate that the symlinks in the archive and
|
||
<code class="docutils literal notranslate"><span class="pre">RECORD</span></code> file are consistent with each other.</p>
|
||
<p>We also considered using <em>only</em> the <code class="docutils literal notranslate"><span class="pre">RECORD</span></code> file to store symlinks,
|
||
but then the vanilla <code class="docutils literal notranslate"><span class="pre">unzip</span></code> tool wouldn’t be able to unpack them, and
|
||
that would make it hard to install a pybi from a shell script.</p>
|
||
</section>
|
||
<section id="limitations">
|
||
<h4><a class="toc-backref" href="#limitations" role="doc-backlink">Limitations</a></h4>
|
||
<p>Symlinks enable a lot of potential messiness. To keep things under
|
||
control, we impose the following restrictions:</p>
|
||
<ul>
|
||
<li>Symlinks MUST NOT be used in <code class="docutils literal notranslate"><span class="pre">.pybi</span></code>s targeting Windows, or other
|
||
platforms that are missing first-class symlink support.</li>
|
||
<li>Symlinks MUST NOT be used inside the <code class="docutils literal notranslate"><span class="pre">pybi-info</span></code> directory.
|
||
(Rationale: there’s no need, and it makes things simpler for
|
||
resolvers that need to extract info from <code class="docutils literal notranslate"><span class="pre">pybi-info</span></code> without
|
||
unpacking the whole archive.)</li>
|
||
<li>Symlink targets MUST be relative paths, and MUST be inside the pybi
|
||
directory.</li>
|
||
<li>If <code class="docutils literal notranslate"><span class="pre">A/B/...</span></code> is recorded as a symlink in the archive, then there
|
||
MUST NOT be any other entries in the archive named like
|
||
<code class="docutils literal notranslate"><span class="pre">A/B/.../C</span></code>.<p>For example, if an archive has a symlink <code class="docutils literal notranslate"><span class="pre">foo</span> <span class="pre">-></span> <span class="pre">bar</span></code>, and then
|
||
later in the archive there’s a regular file named <code class="docutils literal notranslate"><span class="pre">foo/blah.py</span></code>,
|
||
then a naive unpacker could potentially end up writing a file called
|
||
<code class="docutils literal notranslate"><span class="pre">bar/blah.py</span></code>. Don’t be naive.</p>
|
||
</li>
|
||
</ul>
|
||
<p>Unpackers MUST verify that these rules are followed, because without
|
||
them attackers could create evil symlinks like <code class="docutils literal notranslate"><span class="pre">foo</span> <span class="pre">-></span> <span class="pre">/etc/passwd</span></code> or
|
||
<code class="docutils literal notranslate"><span class="pre">foo</span> <span class="pre">-></span> <span class="pre">../../../../../etc</span></code> + <code class="docutils literal notranslate"><span class="pre">foo/passwd</span> <span class="pre">-></span> <span class="pre">...</span></code> and cause havoc.</p>
|
||
</section>
|
||
</section>
|
||
</section>
|
||
<section id="non-normative-comments">
|
||
<h2><a class="toc-backref" href="#non-normative-comments" role="doc-backlink">Non-normative comments</a></h2>
|
||
<section id="why-not-just-use-conda">
|
||
<h3><a class="toc-backref" href="#why-not-just-use-conda" role="doc-backlink">Why not just use conda?</a></h3>
|
||
<p>This isn’t really in the scope of this PEP, but since conda is a popular way to
|
||
distribute binary Python interpreters, it’s a natural question.</p>
|
||
<p>The simple answer is: conda is great! But, there are lots of python users who
|
||
aren’t conda users, and they deserve nice things too. This PEP just gives them
|
||
another option.</p>
|
||
<p>The deeper answer is: the maintainers who upload packages to PyPI are the
|
||
backbone of the Python ecosystem. They’re the first audience for Python
|
||
packaging tools. And one thing they want is to upload a package once, and have
|
||
it be accessible across all the different ways Python is deployed: in Debian and
|
||
Fedora and Homebrew and FreeBSD, in Conda environments, in big companies’
|
||
monorepos, in Nix, in Blender plugins, in RenPy games, ….. you get the idea.</p>
|
||
<p>All of these environments have their own tooling and strategies for managing
|
||
packages and dependencies. So what’s special about PyPI and wheels is that
|
||
they’re designed to describe dependencies in a <em>standard, abstract way</em>, that
|
||
all these downstream systems can consume and convert into their local
|
||
conventions. That’s why package maintainers use Python-specific metadata and
|
||
upload to PyPI: because it lets them address all of those systems
|
||
simultaneously. Every time you build a Python package for conda, there’s an
|
||
intermediate wheel that’s generated, because wheels are the common language that
|
||
Python package build systems and conda can use to talk to each other.</p>
|
||
<p>But then, if you’re a maintainer releasing an sdist+wheels, then you naturally
|
||
want to test what you’re releasing, which may depend on arbitrary PyPI packages
|
||
and versions. So you need tools that build Python environments directly from
|
||
PyPI, and conda is fundamentally not designed to do that. So conda and pip are
|
||
both necessary for different cases, and this proposal happens to be targeting
|
||
the pip side of that equation.</p>
|
||
</section>
|
||
<section id="sdists-or-not">
|
||
<h3><a class="toc-backref" href="#sdists-or-not" role="doc-backlink">Sdists (or not)</a></h3>
|
||
<p>It might be cool to have an “sdist” equivalent for pybis, i.e., some
|
||
kind of format for a Python source release that’s structured-enough to
|
||
let tools automatically fetch and build it into a pybi, for platforms
|
||
where prebuilt pybis aren’t available. But, this isn’t necessary for the
|
||
MVP and opens a can of worms, so let’s worry about it later.</p>
|
||
</section>
|
||
<section id="what-packages-should-be-bundled-inside-a-pybi">
|
||
<h3><a class="toc-backref" href="#what-packages-should-be-bundled-inside-a-pybi" role="doc-backlink">What packages should be bundled inside a pybi?</a></h3>
|
||
<p>Pybi builders have the power to pick and choose what exactly goes inside. For
|
||
example, you could include some preinstalled packages in the pybi’s
|
||
<code class="docutils literal notranslate"><span class="pre">site-packages</span></code> directory, or prune out bits of the stdlib that you don’t
|
||
want. We can’t stop you! Though if you do preinstall packages, then it’s
|
||
strongly recommended to also include the correct metadata (<code class="docutils literal notranslate"><span class="pre">.dist-info</span></code> etc.),
|
||
so that it’s possible for Pip or other tools to understand out what’s going on.</p>
|
||
<p>For my prototype “general purpose” pybi’s, what I chose is:</p>
|
||
<ul>
|
||
<li>Make sure <code class="docutils literal notranslate"><span class="pre">site-packages</span></code> is <em>empty</em>.<p><strong>Rationale:</strong> for traditional standalone python installers that are targeted
|
||
at end-users, you probably want to include at least <code class="docutils literal notranslate"><span class="pre">pip</span></code>, to avoid
|
||
bootstrapping issues (<a class="pep reference internal" href="../pep-0453/" title="PEP 453 – Explicit bootstrapping of pip in Python installations">PEP 453</a>). But pybis are different: they’re designed
|
||
to be installed by “smart” tooling, that consume the pybi as part of some
|
||
kind of larger automated deployment process. It’s easier for these installers
|
||
to start from a blank slate and then add whatever they need, than for them to
|
||
start with some preinstalled packages that they may or may not want. (And
|
||
besides, you can still run <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-m</span> <span class="pre">ensurepip</span></code>.)</p>
|
||
</li>
|
||
<li>Include the full stdlib, <em>except</em> for <code class="docutils literal notranslate"><span class="pre">test</span></code>.<p><strong>Rationale:</strong> the top-level <code class="docutils literal notranslate"><span class="pre">test</span></code> module contains CPython’s own test
|
||
suite. It’s huge (CPython without <code class="docutils literal notranslate"><span class="pre">test</span></code> is ~37 MB, then <code class="docutils literal notranslate"><span class="pre">test</span></code>
|
||
adds another ~25 MB on top of that!), and essentially never used by
|
||
regular user code. Also, as precedent, the official nuget packages,
|
||
the official manylinux images, and multiple Linux distributions all
|
||
leave it out, and this hasn’t caused any major problems.</p>
|
||
<p>So this seems like the best way to balance broad compatibility with
|
||
reasonable download/install sizes.</p>
|
||
</li>
|
||
<li>I’m not shipping any <code class="docutils literal notranslate"><span class="pre">.pyc</span></code> files. They take up space in the
|
||
download, can be generated on the final system at minimal cost, and
|
||
dropping them removes a source of location-dependence. (<code class="docutils literal notranslate"><span class="pre">.pyc</span></code>
|
||
files store the absolute path of the corresponding <code class="docutils literal notranslate"><span class="pre">.py</span></code> file and
|
||
include it in tracebacks; but, pybis are relocatable, so the correct
|
||
path isn’t known until after install.)</li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards Compatibility</a></h2>
|
||
<p>No backwards compatibility considerations.</p>
|
||
</section>
|
||
<section id="security-implications">
|
||
<h2><a class="toc-backref" href="#security-implications" role="doc-backlink">Security Implications</a></h2>
|
||
<p>No security implications, beyond the fact that anyone who takes it upon
|
||
themselves to distribute binaries has to come up with a plan to manage their
|
||
security (e.g., whether they roll a new build after an OpenSSL CVE drops). But
|
||
collectively, we core Python folks are already maintaining binary builds for all
|
||
major platforms (macOS + Windows through python.org, and Linux builds through
|
||
the official manylinux image), so even if we do start releasing official CPython
|
||
builds on PyPI it doesn’t really raise any new security issues.</p>
|
||
</section>
|
||
<section id="how-to-teach-this">
|
||
<h2><a class="toc-backref" href="#how-to-teach-this" role="doc-backlink">How to Teach This</a></h2>
|
||
<p>This isn’t targeted at end-users; their experience will simply be that e.g.
|
||
their pyenv or tox invocation magically gets faster and more reliable (if those
|
||
projects’ maintainers decide to take advantage of this PEP).</p>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document is placed in the public domain or under the
|
||
CC0-1.0-Universal license, whichever is more permissive.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0711.rst">https://github.com/python/peps/blob/main/peps/pep-0711.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0711.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#examples">Examples</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#filename">Filename</a></li>
|
||
<li><a class="reference internal" href="#file-contents">File contents</a><ul>
|
||
<li><a class="reference internal" href="#pybi-specific-core-metadata">Pybi-specific core metadata</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#symlinks">Symlinks</a><ul>
|
||
<li><a class="reference internal" href="#representing-symlinks-in-zip-files">Representing symlinks in zip files</a></li>
|
||
<li><a class="reference internal" href="#representing-symlinks-in-record-files">Representing symlinks in RECORD files</a></li>
|
||
<li><a class="reference internal" href="#storing-symlinks-in-pybi-files">Storing symlinks in <code class="docutils literal notranslate"><span class="pre">pybi</span></code> files</a></li>
|
||
<li><a class="reference internal" href="#limitations">Limitations</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#non-normative-comments">Non-normative comments</a><ul>
|
||
<li><a class="reference internal" href="#why-not-just-use-conda">Why not just use conda?</a></li>
|
||
<li><a class="reference internal" href="#sdists-or-not">Sdists (or not)</a></li>
|
||
<li><a class="reference internal" href="#what-packages-should-be-bundled-inside-a-pybi">What packages should be bundled inside a pybi?</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#security-implications">Security Implications</a></li>
|
||
<li><a class="reference internal" href="#how-to-teach-this">How to Teach This</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0711.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |