mirror of https://github.com/python/peps
761 lines
61 KiB
HTML
761 lines
61 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 3147 – PYC Repository Directories | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-3147/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 3147 – PYC Repository Directories | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-3147/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 3147</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 3147 – PYC Repository Directories</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Barry Warsaw <barry at python.org></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Accepted and implementation complete, or no longer active">Final</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">16-Dec-2009</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.2</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">30-Jan-2010, 25-Feb-2010, 03-Mar-2010, 12-Apr-2010</dd>
|
||
<dt class="field-odd">Resolution<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://mail.python.org/pipermail/python-dev/2010-April/099414.html">Python-Dev message</a></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#background">Background</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a><ul>
|
||
<li><a class="reference internal" href="#examples">Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#python-behavior">Python behavior</a><ul>
|
||
<li><a class="reference internal" href="#case-0-the-steady-state">Case 0: The steady state</a></li>
|
||
<li><a class="reference internal" href="#case-1-the-first-import">Case 1: The first import</a></li>
|
||
<li><a class="reference internal" href="#case-2-the-second-import">Case 2: The second import</a></li>
|
||
<li><a class="reference internal" href="#case-3-pycache-foo-magic-pyc-with-no-source">Case 3: __pycache__/foo.<magic>.pyc with no source</a></li>
|
||
<li><a class="reference internal" href="#case-4-legacy-pyc-files-and-source-less-imports">Case 4: legacy pyc files and source-less imports</a></li>
|
||
<li><a class="reference internal" href="#case-5-read-only-file-systems">Case 5: read-only file systems</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#flow-chart">Flow chart</a></li>
|
||
<li><a class="reference internal" href="#alternative-python-implementations">Alternative Python implementations</a></li>
|
||
<li><a class="reference internal" href="#implementation-strategy">Implementation strategy</a></li>
|
||
<li><a class="reference internal" href="#effects-on-existing-code">Effects on existing code</a><ul>
|
||
<li><a class="reference internal" href="#detecting-pep-3147-availability">Detecting PEP 3147 availability</a></li>
|
||
<li><a class="reference internal" href="#file">__file__</a></li>
|
||
<li><a class="reference internal" href="#py-compile-and-compileall">py_compile and compileall</a></li>
|
||
<li><a class="reference internal" href="#bdist-wininst-and-the-windows-installer">bdist_wininst and the Windows installer</a></li>
|
||
<li><a class="reference internal" href="#file-extension-checks">File extension checks</a></li>
|
||
<li><a class="reference internal" href="#backports">Backports</a></li>
|
||
<li><a class="reference internal" href="#makefiles-and-other-dependency-tools">Makefiles and other dependency tools</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#alternatives">Alternatives</a><ul>
|
||
<li><a class="reference internal" href="#hexadecimal-magic-tags">Hexadecimal magic tags</a></li>
|
||
<li><a class="reference internal" href="#pep-304">PEP 304</a></li>
|
||
<li><a class="reference internal" href="#fat-byte-compilation-files">Fat byte compilation files</a></li>
|
||
<li><a class="reference internal" href="#multiple-file-extensions">Multiple file extensions</a></li>
|
||
<li><a class="reference internal" href="#pyc">.pyc</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#acknowledgments">ACKNOWLEDGMENTS</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP describes an extension to Python’s import mechanism which
|
||
improves sharing of Python source code files among multiple installed
|
||
different versions of the Python interpreter. It does this by
|
||
allowing more than one byte compilation file (.pyc files) to be
|
||
co-located with the Python source file (.py file). The extension
|
||
described here can also be used to support different Python
|
||
compilation caches, such as JIT output that may be produced by an
|
||
Unladen Swallow (<a class="pep reference internal" href="../pep-3146/" title="PEP 3146 – Merging Unladen Swallow into CPython">PEP 3146</a>) enabled C Python.</p>
|
||
</section>
|
||
<section id="background">
|
||
<h2><a class="toc-backref" href="#background" role="doc-backlink">Background</a></h2>
|
||
<p>CPython compiles its source code into “byte code”, and for performance
|
||
reasons, it caches this byte code on the file system whenever the
|
||
source file has changes. This makes loading of Python modules much
|
||
faster because the compilation phase can be bypassed. When your
|
||
source file is <code class="docutils literal notranslate"><span class="pre">foo.py</span></code>, CPython caches the byte code in a <code class="docutils literal notranslate"><span class="pre">foo.pyc</span></code>
|
||
file right next to the source.</p>
|
||
<p>Byte code files contain two 32-bit big-endian numbers followed by the
|
||
marshaled <a class="footnote-reference brackets" href="#id22" id="id1">[2]</a> code object. The 32-bit numbers represent a magic
|
||
number and a timestamp. The magic number changes whenever Python
|
||
changes the byte code format, e.g. by adding new byte codes to its
|
||
virtual machine. This ensures that pyc files built for previous
|
||
versions of the VM won’t cause problems. The timestamp is used to
|
||
make sure that the pyc file match the py file that was used to create
|
||
it. When either the magic number or timestamp do not match, the py
|
||
file is recompiled and a new pyc file is written.</p>
|
||
<p>In practice, it is well known that pyc files are not compatible across
|
||
Python major releases. A reading of import.c <a class="footnote-reference brackets" href="#id23" id="id2">[3]</a> in the Python
|
||
source code proves that within recent memory, every new CPython major
|
||
release has bumped the pyc magic number.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Linux distributions such as Ubuntu <a class="footnote-reference brackets" href="#id24" id="id3">[4]</a> and Debian <a class="footnote-reference brackets" href="#id25" id="id4">[5]</a> provide more
|
||
than one Python version at the same time to their users. For example,
|
||
Ubuntu 9.10 Karmic Koala users can install Python 2.5, 2.6, and 3.1,
|
||
with Python 2.6 being the default.</p>
|
||
<p>This causes a conflict for third party Python source files installed
|
||
by the system, because you cannot compile a single Python source file
|
||
for more than one Python version at a time. When Python finds a <code class="docutils literal notranslate"><span class="pre">pyc</span></code>
|
||
file with a non-matching magic number, it falls back to the slower
|
||
process of recompiling the source. Thus if your system installed a
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/share/python/foo.py</span></code>, two different versions of Python would
|
||
fight over the <code class="docutils literal notranslate"><span class="pre">pyc</span></code> file and rewrite it each time the source is
|
||
compiled. (The standard library is unaffected by this, since multiple
|
||
versions of the stdlib <em>are</em> installed on such distributions..)</p>
|
||
<p>Furthermore, in order to ease the burden on operating system packagers
|
||
for these distributions, the distribution packages do not contain
|
||
Python version numbers <a class="footnote-reference brackets" href="#id26" id="id5">[6]</a>; they are shared across all Python
|
||
versions installed on the system. Putting Python version numbers in
|
||
the packages would be a maintenance nightmare, since all the packages
|
||
- <em>and their dependencies</em> - would have to be updated every time a new
|
||
Python release was added or removed from the distribution. Because of
|
||
the sheer number of packages available, this amount of work is
|
||
infeasible.</p>
|
||
<p>(<a class="pep reference internal" href="../pep-0384/" title="PEP 384 – Defining a Stable ABI">PEP 384</a> has been proposed to address binary compatibility issues
|
||
of third party extension modules across different versions of Python.)</p>
|
||
<p>Because these distributions cannot share pyc files, elaborate
|
||
mechanisms have been developed to put the resulting pyc files in
|
||
non-shared locations while the source code is still shared. Examples
|
||
include the symlink-based Debian regimes python-support <a class="footnote-reference brackets" href="#id27" id="id6">[8]</a> and
|
||
python-central <a class="footnote-reference brackets" href="#id28" id="id7">[9]</a>. These approaches make for much more complicated,
|
||
fragile, inscrutable, and fragmented policies for delivering Python
|
||
applications to a wide range of users. Arguably more users get Python
|
||
from their operating system vendor than from upstream tarballs. Thus,
|
||
solving this pyc sharing problem for CPython is a high priority for
|
||
such vendors.</p>
|
||
<p>This PEP proposes a solution to this problem.</p>
|
||
</section>
|
||
<section id="proposal">
|
||
<h2><a class="toc-backref" href="#proposal" role="doc-backlink">Proposal</a></h2>
|
||
<p>Python’s import machinery is extended to write and search for byte
|
||
code cache files in a single directory inside every Python package
|
||
directory. This directory will be called <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code>.</p>
|
||
<p>Further, pyc file names will contain a magic string (called a “tag”)
|
||
that differentiates the Python version they were compiled for. This
|
||
allows multiple byte compiled cache files to co-exist for a single
|
||
Python source file.</p>
|
||
<p>The magic tag is implementation defined, but should contain the
|
||
implementation name and a version number shorthand, e.g. <code class="docutils literal notranslate"><span class="pre">cpython-32</span></code>.
|
||
It must be unique among all versions of Python, and whenever the magic
|
||
number is bumped, a new magic tag must be defined. An example <code class="docutils literal notranslate"><span class="pre">pyc</span></code>
|
||
file for Python 3.2 is thus <code class="docutils literal notranslate"><span class="pre">foo.cpython-32.pyc</span></code>.</p>
|
||
<p>The magic tag is available in the <code class="docutils literal notranslate"><span class="pre">imp</span></code> module via the <code class="docutils literal notranslate"><span class="pre">get_tag()</span></code>
|
||
function. This is parallel to the <code class="docutils literal notranslate"><span class="pre">imp.get_magic()</span></code> function.</p>
|
||
<p>This scheme has the added benefit of reducing the clutter in a Python
|
||
package directory.</p>
|
||
<p>When a Python source file is imported for the first time, a
|
||
<code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directory will be created in the package directory, if
|
||
one does not already exist. The pyc file for the imported source will
|
||
be written to the <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directory, using the magic-tag
|
||
formatted name. If either the creation of the <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directory
|
||
or the pyc file inside that fails, the import will still succeed, just
|
||
as it does in a pre-<a class="pep reference internal" href="../pep-3147/" title="PEP 3147 – PYC Repository Directories">PEP 3147</a> world.</p>
|
||
<p>If the py source file is missing, the pyc file inside <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code>
|
||
will be ignored. This eliminates the problem of accidental stale pyc
|
||
file imports.</p>
|
||
<p>For backward compatibility, Python will still support pyc-only
|
||
distributions, however it will only do so when the pyc file lives in
|
||
the directory where the py file <em>would</em> have been, i.e. not in the
|
||
<code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directory. pyc file outside of <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> will only
|
||
be imported if the py source file is missing.</p>
|
||
<p>Tools such as <code class="docutils literal notranslate"><span class="pre">py_compile</span></code> <a class="footnote-reference brackets" href="#id34" id="id8">[15]</a> and <code class="docutils literal notranslate"><span class="pre">compileall</span></code> <a class="footnote-reference brackets" href="#id35" id="id9">[16]</a> will be
|
||
extended to create <a class="pep reference internal" href="../pep-3147/" title="PEP 3147 – PYC Repository Directories">PEP 3147</a> formatted layouts automatically, but will
|
||
have an option to create pyc-only distribution layouts.</p>
|
||
<section id="examples">
|
||
<h3><a class="toc-backref" href="#examples" role="doc-backlink">Examples</a></h3>
|
||
<p>What would this look like in practice?</p>
|
||
<p>Let’s say we have a Python package named <code class="docutils literal notranslate"><span class="pre">alpha</span></code> which contains a
|
||
sub-package name <code class="docutils literal notranslate"><span class="pre">beta</span></code>. The source directory layout before byte
|
||
compilation might look like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">alpha</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">beta</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">py</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>After byte compiling this package with Python 3.2, you would see the
|
||
following layout:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">alpha</span><span class="o">/</span>
|
||
<span class="n">__pycache__</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">beta</span><span class="o">/</span>
|
||
<span class="n">__pycache__</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">py</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><em>Note: listing order may differ depending on the platform.</em></p>
|
||
<p>Let’s say that two new versions of Python are installed, one is Python
|
||
3.3 and another is Unladen Swallow. After byte compilation, the file
|
||
system would look like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">alpha</span><span class="o">/</span>
|
||
<span class="n">__pycache__</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">33.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">unladen</span><span class="o">-</span><span class="mf">10.</span><span class="n">pyc</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">33.</span><span class="n">pyc</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">unladen</span><span class="o">-</span><span class="mf">10.</span><span class="n">pyc</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">33.</span><span class="n">pyc</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">unladen</span><span class="o">-</span><span class="mf">10.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">beta</span><span class="o">/</span>
|
||
<span class="n">__pycache__</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">33.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">unladen</span><span class="o">-</span><span class="mf">10.</span><span class="n">pyc</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">33.</span><span class="n">pyc</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">unladen</span><span class="o">-</span><span class="mf">10.</span><span class="n">pyc</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">32.</span><span class="n">pyc</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">cpython</span><span class="o">-</span><span class="mf">33.</span><span class="n">pyc</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">unladen</span><span class="o">-</span><span class="mf">10.</span><span class="n">pyc</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">py</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>As you can see, as long as the Python version identifier string is
|
||
unique, any number of pyc files can co-exist. These identifier
|
||
strings are described in more detail below.</p>
|
||
<p>A nice property of this layout is that the <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directories
|
||
can generally be ignored, such that a normal directory listing would
|
||
show something like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">alpha</span><span class="o">/</span>
|
||
<span class="n">__pycache__</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">one</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">two</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">beta</span><span class="o">/</span>
|
||
<span class="n">__pycache__</span><span class="o">/</span>
|
||
<span class="fm">__init__</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">three</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">four</span><span class="o">.</span><span class="n">py</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This is much less cluttered than even today’s Python.</p>
|
||
</section>
|
||
</section>
|
||
<section id="python-behavior">
|
||
<h2><a class="toc-backref" href="#python-behavior" role="doc-backlink">Python behavior</a></h2>
|
||
<p>When Python searches for a module to import (say <code class="docutils literal notranslate"><span class="pre">foo</span></code>), it may find
|
||
one of several situations. As per current Python rules, the term
|
||
“matching pyc” means that the magic number matches the current
|
||
interpreter’s magic number, and the source file’s timestamp matches
|
||
the timestamp in the <code class="docutils literal notranslate"><span class="pre">pyc</span></code> file exactly.</p>
|
||
<section id="case-0-the-steady-state">
|
||
<h3><a class="toc-backref" href="#case-0-the-steady-state" role="doc-backlink">Case 0: The steady state</a></h3>
|
||
<p>When Python is asked to import module <code class="docutils literal notranslate"><span class="pre">foo</span></code>, it searches for a
|
||
<code class="docutils literal notranslate"><span class="pre">foo.py</span></code> file (or <code class="docutils literal notranslate"><span class="pre">foo</span></code> package, but that’s not important for this
|
||
discussion) along its <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>. If found, Python looks to see if
|
||
there is a matching <code class="docutils literal notranslate"><span class="pre">__pycache__/foo.<magic>.pyc</span></code> file, and if so,
|
||
that <code class="docutils literal notranslate"><span class="pre">pyc</span></code> file is loaded.</p>
|
||
</section>
|
||
<section id="case-1-the-first-import">
|
||
<h3><a class="toc-backref" href="#case-1-the-first-import" role="doc-backlink">Case 1: The first import</a></h3>
|
||
<p>When Python locates the <code class="docutils literal notranslate"><span class="pre">foo.py</span></code>, if the <code class="docutils literal notranslate"><span class="pre">__pycache__/foo.<magic>.pyc</span></code>
|
||
file is missing, Python will create it, also creating the
|
||
<code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directory if necessary. Python will parse and byte
|
||
compile the <code class="docutils literal notranslate"><span class="pre">foo.py</span></code> file and save the byte code in
|
||
<code class="docutils literal notranslate"><span class="pre">__pycache__/foo.<magic>.pyc</span></code>.</p>
|
||
</section>
|
||
<section id="case-2-the-second-import">
|
||
<h3><a class="toc-backref" href="#case-2-the-second-import" role="doc-backlink">Case 2: The second import</a></h3>
|
||
<p>When Python is asked to import module <code class="docutils literal notranslate"><span class="pre">foo</span></code> a second time (in a
|
||
different process of course), it will again search for the <code class="docutils literal notranslate"><span class="pre">foo.py</span></code>
|
||
file along its <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>. When Python locates the <code class="docutils literal notranslate"><span class="pre">foo.py</span></code> file, it
|
||
looks for a matching <code class="docutils literal notranslate"><span class="pre">__pycache__/foo.<magic>.pyc</span></code> and finding this,
|
||
it reads the byte code and continues as usual.</p>
|
||
</section>
|
||
<section id="case-3-pycache-foo-magic-pyc-with-no-source">
|
||
<h3><a class="toc-backref" href="#case-3-pycache-foo-magic-pyc-with-no-source" role="doc-backlink">Case 3: __pycache__/foo.<magic>.pyc with no source</a></h3>
|
||
<p>It’s possible that the <code class="docutils literal notranslate"><span class="pre">foo.py</span></code> file somehow got removed, while
|
||
leaving the cached pyc file still on the file system. If the
|
||
<code class="docutils literal notranslate"><span class="pre">__pycache__/foo.<magic>.pyc</span></code> file exists, but the <code class="docutils literal notranslate"><span class="pre">foo.py</span></code> file used
|
||
to create it does not, Python will raise an <code class="docutils literal notranslate"><span class="pre">ImportError</span></code> when asked
|
||
to import foo. In other words, Python will not import a pyc file from
|
||
the cache directory unless the source file exists.</p>
|
||
</section>
|
||
<section id="case-4-legacy-pyc-files-and-source-less-imports">
|
||
<h3><a class="toc-backref" href="#case-4-legacy-pyc-files-and-source-less-imports" role="doc-backlink">Case 4: legacy pyc files and source-less imports</a></h3>
|
||
<p>Python will ignore all legacy pyc files when a source file exists next
|
||
to it. In other words, if a <code class="docutils literal notranslate"><span class="pre">foo.pyc</span></code> file exists next to the
|
||
<code class="docutils literal notranslate"><span class="pre">foo.py</span></code> file, the pyc file will be ignored in all cases</p>
|
||
<p>In order to continue to support source-less distributions though, if
|
||
the source file is missing, Python will import a lone pyc file if it
|
||
lives where the source file would have been.</p>
|
||
</section>
|
||
<section id="case-5-read-only-file-systems">
|
||
<h3><a class="toc-backref" href="#case-5-read-only-file-systems" role="doc-backlink">Case 5: read-only file systems</a></h3>
|
||
<p>When the source lives on a read-only file system, or the <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code>
|
||
directory or pyc file cannot otherwise be written, all the same rules
|
||
apply. This is also the case when <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> happens to be written
|
||
with permissions which do not allow for writing containing pyc files.</p>
|
||
</section>
|
||
</section>
|
||
<section id="flow-chart">
|
||
<h2><a class="toc-backref" href="#flow-chart" role="doc-backlink">Flow chart</a></h2>
|
||
<p>Here is a flow chart describing how modules are loaded:</p>
|
||
<a class="invert-in-dark-mode reference internal image-reference" href="../_images/pep-3147-1.png"><img alt="../_images/pep-3147-1.png" class="invert-in-dark-mode" src="../_images/pep-3147-1.png" style="width: 777.75px; height: 609.0px;" /></a>
|
||
</section>
|
||
<section id="alternative-python-implementations">
|
||
<h2><a class="toc-backref" href="#alternative-python-implementations" role="doc-backlink">Alternative Python implementations</a></h2>
|
||
<p>Alternative Python implementations such as Jython <a class="footnote-reference brackets" href="#id30" id="id10">[11]</a>, IronPython
|
||
<a class="footnote-reference brackets" href="#id31" id="id11">[12]</a>, PyPy <a class="footnote-reference brackets" href="#id32" id="id12">[13]</a>, Pynie <a class="footnote-reference brackets" href="#id33" id="id13">[14]</a>, and Unladen Swallow can also use the
|
||
<code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directory to store whatever compilation artifacts make
|
||
sense for their platforms. For example, Jython could store the class
|
||
file for the module in <code class="docutils literal notranslate"><span class="pre">__pycache__/foo.jython-32.class</span></code>.</p>
|
||
</section>
|
||
<section id="implementation-strategy">
|
||
<h2><a class="toc-backref" href="#implementation-strategy" role="doc-backlink">Implementation strategy</a></h2>
|
||
<p>This feature is targeted for Python 3.2, solving the problem for those
|
||
and all future versions. It may be back-ported to Python 2.7.
|
||
Vendors are free to backport the changes to earlier distributions as
|
||
they see fit. For backports of this feature to Python 2, when the
|
||
<code class="docutils literal notranslate"><span class="pre">-U</span></code> flag is used, a file such as <code class="docutils literal notranslate"><span class="pre">foo.cpython-27u.pyc</span></code> can be
|
||
written.</p>
|
||
</section>
|
||
<section id="effects-on-existing-code">
|
||
<h2><a class="toc-backref" href="#effects-on-existing-code" role="doc-backlink">Effects on existing code</a></h2>
|
||
<p>Adoption of this PEP will affect existing code and idioms, both inside
|
||
Python and outside. This section enumerates some of these effects.</p>
|
||
<section id="detecting-pep-3147-availability">
|
||
<h3><a class="toc-backref" href="#detecting-pep-3147-availability" role="doc-backlink">Detecting PEP 3147 availability</a></h3>
|
||
<p>The easiest way to detect whether your version of Python provides PEP
|
||
3147 functionality is to do the following check:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">imp</span>
|
||
<span class="gp">>>> </span><span class="n">has3147</span> <span class="o">=</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">imp</span><span class="p">,</span> <span class="s1">'get_tag'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="file">
|
||
<h3><a class="toc-backref" href="#file" role="doc-backlink">__file__</a></h3>
|
||
<p>In Python 3, when you import a module, its <code class="docutils literal notranslate"><span class="pre">__file__</span></code> attribute points
|
||
to its source <code class="docutils literal notranslate"><span class="pre">py</span></code> file (in Python 2, it points to the <code class="docutils literal notranslate"><span class="pre">pyc</span></code> file). A
|
||
package’s <code class="docutils literal notranslate"><span class="pre">__file__</span></code> points to the <code class="docutils literal notranslate"><span class="pre">py</span></code> file for its <code class="docutils literal notranslate"><span class="pre">__init__.py</span></code>.
|
||
E.g.:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">foo</span>
|
||
<span class="gp">>>> </span><span class="n">foo</span><span class="o">.</span><span class="vm">__file__</span>
|
||
<span class="go">'foo.py'</span>
|
||
<span class="go"># baz is a package</span>
|
||
<span class="gp">>>> </span><span class="kn">import</span> <span class="nn">baz</span>
|
||
<span class="gp">>>> </span><span class="n">baz</span><span class="o">.</span><span class="vm">__file__</span>
|
||
<span class="go">'baz/__init__.py'</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Nothing in this PEP would change the semantics of <code class="docutils literal notranslate"><span class="pre">__file__</span></code>.</p>
|
||
<p>This PEP proposes the addition of an <code class="docutils literal notranslate"><span class="pre">__cached__</span></code> attribute to
|
||
modules, which will always point to the actual <code class="docutils literal notranslate"><span class="pre">pyc</span></code> file that was
|
||
read or written. When the environment variable
|
||
<code class="docutils literal notranslate"><span class="pre">$PYTHONDONTWRITEBYTECODE</span></code> is set, or the <code class="docutils literal notranslate"><span class="pre">-B</span></code> option is given, or if
|
||
the source lives on a read-only filesystem, then the <code class="docutils literal notranslate"><span class="pre">__cached__</span></code>
|
||
attribute will point to the location that the <code class="docutils literal notranslate"><span class="pre">pyc</span></code> file <em>would</em> have
|
||
been written to if it didn’t exist. This location of course includes
|
||
the <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> subdirectory in its path.</p>
|
||
<p>For alternative Python implementations which do not support <code class="docutils literal notranslate"><span class="pre">pyc</span></code>
|
||
files, the <code class="docutils literal notranslate"><span class="pre">__cached__</span></code> attribute may point to whatever information
|
||
makes sense. E.g. on Jython, this might be the <code class="docutils literal notranslate"><span class="pre">.class</span></code> file for the
|
||
module: <code class="docutils literal notranslate"><span class="pre">__pycache__/foo.jython-32.class</span></code>. Some implementations may
|
||
use multiple compiled files to create the module, in which case
|
||
<code class="docutils literal notranslate"><span class="pre">__cached__</span></code> may be a tuple. The exact contents of <code class="docutils literal notranslate"><span class="pre">__cached__</span></code> are
|
||
Python implementation specific.</p>
|
||
<p>It is recommended that when nothing sensible can be calculated,
|
||
implementations should set the <code class="docutils literal notranslate"><span class="pre">__cached__</span></code> attribute to <code class="docutils literal notranslate"><span class="pre">None</span></code>.</p>
|
||
</section>
|
||
<section id="py-compile-and-compileall">
|
||
<h3><a class="toc-backref" href="#py-compile-and-compileall" role="doc-backlink">py_compile and compileall</a></h3>
|
||
<p>Python comes with two modules, <code class="docutils literal notranslate"><span class="pre">py_compile</span></code> <a class="footnote-reference brackets" href="#id34" id="id14">[15]</a> and <code class="docutils literal notranslate"><span class="pre">compileall</span></code>
|
||
<a class="footnote-reference brackets" href="#id35" id="id15">[16]</a> which support compiling Python modules external to the built-in
|
||
import machinery. <code class="docutils literal notranslate"><span class="pre">py_compile</span></code> in particular has intimate knowledge
|
||
of byte compilation, so these will be updated to understand the new
|
||
layout. The <code class="docutils literal notranslate"><span class="pre">-b</span></code> flag is added to <code class="docutils literal notranslate"><span class="pre">compileall</span></code> for writing legacy
|
||
<code class="docutils literal notranslate"><span class="pre">.pyc</span></code> byte-compiled file path names.</p>
|
||
</section>
|
||
<section id="bdist-wininst-and-the-windows-installer">
|
||
<h3><a class="toc-backref" href="#bdist-wininst-and-the-windows-installer" role="doc-backlink">bdist_wininst and the Windows installer</a></h3>
|
||
<p>These tools also compile modules explicitly on installation. If they
|
||
do not use <code class="docutils literal notranslate"><span class="pre">py_compile</span></code> and <code class="docutils literal notranslate"><span class="pre">compileall</span></code>, then they would also have to
|
||
be modified to understand the new layout.</p>
|
||
</section>
|
||
<section id="file-extension-checks">
|
||
<h3><a class="toc-backref" href="#file-extension-checks" role="doc-backlink">File extension checks</a></h3>
|
||
<p>There exists some code which checks for files ending in <code class="docutils literal notranslate"><span class="pre">.pyc</span></code> and
|
||
simply chops off the last character to find the matching <code class="docutils literal notranslate"><span class="pre">.py</span></code> file.
|
||
This code will obviously fail once this PEP is implemented.</p>
|
||
<p>To support this use case, we’ll add two new methods to the <code class="docutils literal notranslate"><span class="pre">imp</span></code>
|
||
package <a class="footnote-reference brackets" href="#id36" id="id16">[17]</a>:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">imp.cache_from_source(py_path)</span></code> -> <code class="docutils literal notranslate"><span class="pre">pyc_path</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">imp.source_from_cache(pyc_path)</span></code> -> <code class="docutils literal notranslate"><span class="pre">py_path</span></code></li>
|
||
</ul>
|
||
<p>Alternative implementations are free to override these functions to
|
||
return reasonable values based on their own support for this PEP.
|
||
These methods are allowed to return <code class="docutils literal notranslate"><span class="pre">None</span></code> when the implementation (or
|
||
<a class="pep reference internal" href="../pep-0302/" title="PEP 302 – New Import Hooks">PEP 302</a> loader in effect) for whatever reason cannot calculate
|
||
the appropriate file name. They should not raise exceptions.</p>
|
||
</section>
|
||
<section id="backports">
|
||
<h3><a class="toc-backref" href="#backports" role="doc-backlink">Backports</a></h3>
|
||
<p>For versions of Python earlier than 3.2 (and possibly 2.7), it is
|
||
possible to backport this PEP. However, in Python 3.2 (and possibly
|
||
2.7), this behavior will be turned on by default, and in fact, it will
|
||
replace the old behavior. Backports will need to support the old
|
||
layout by default. We suggest supporting <a class="pep reference internal" href="../pep-3147/" title="PEP 3147 – PYC Repository Directories">PEP 3147</a> through the use of
|
||
an environment variable called <code class="docutils literal notranslate"><span class="pre">$PYTHONENABLECACHEDIR</span></code> or the command
|
||
line switch <code class="docutils literal notranslate"><span class="pre">-Xenablecachedir</span></code> to enable the feature.</p>
|
||
</section>
|
||
<section id="makefiles-and-other-dependency-tools">
|
||
<h3><a class="toc-backref" href="#makefiles-and-other-dependency-tools" role="doc-backlink">Makefiles and other dependency tools</a></h3>
|
||
<p>Makefiles and other tools which calculate dependencies on <code class="docutils literal notranslate"><span class="pre">.pyc</span></code> files
|
||
(e.g. to byte-compile the source if the <code class="docutils literal notranslate"><span class="pre">.pyc</span></code> is missing) will have
|
||
to be updated to check the new paths.</p>
|
||
</section>
|
||
</section>
|
||
<section id="alternatives">
|
||
<h2><a class="toc-backref" href="#alternatives" role="doc-backlink">Alternatives</a></h2>
|
||
<p>This section describes some alternative approaches or details that
|
||
were considered and rejected during the PEP’s development.</p>
|
||
<section id="hexadecimal-magic-tags">
|
||
<h3><a class="toc-backref" href="#hexadecimal-magic-tags" role="doc-backlink">Hexadecimal magic tags</a></h3>
|
||
<p>pyc files inside of the <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directories contain a magic tag
|
||
in their file names. These are mnemonic tags for the actual magic
|
||
numbers used by the importer. We could have used the hexadecimal
|
||
representation <a class="footnote-reference brackets" href="#id29" id="id17">[10]</a> of the binary magic number as a unique
|
||
identifier. For example, in Python 3.2:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">binascii</span> <span class="kn">import</span> <span class="n">hexlify</span>
|
||
<span class="gp">>>> </span><span class="kn">from</span> <span class="nn">imp</span> <span class="kn">import</span> <span class="n">get_magic</span>
|
||
<span class="gp">>>> </span><span class="s1">'foo.</span><span class="si">{}</span><span class="s1">.pyc'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hexlify</span><span class="p">(</span><span class="n">get_magic</span><span class="p">())</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'ascii'</span><span class="p">))</span>
|
||
<span class="go">'foo.580c0d0a.pyc'</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This isn’t particularly human friendly though, thus the magic tag
|
||
proposed in this PEP.</p>
|
||
</section>
|
||
<section id="pep-304">
|
||
<h3><a class="toc-backref" href="#pep-304" role="doc-backlink">PEP 304</a></h3>
|
||
<p>There is some overlap between the goals of this PEP and <a class="pep reference internal" href="../pep-0304/" title="PEP 304 – Controlling Generation of Bytecode Files">PEP 304</a>,
|
||
which has been withdrawn. However <a class="pep reference internal" href="../pep-0304/" title="PEP 304 – Controlling Generation of Bytecode Files">PEP 304</a> would allow a user to
|
||
create a shadow file system hierarchy in which to store <code class="docutils literal notranslate"><span class="pre">pyc</span></code> files.
|
||
This concept of a shadow hierarchy for <code class="docutils literal notranslate"><span class="pre">pyc</span></code> files could be used to
|
||
satisfy the aims of this PEP. Although the <a class="pep reference internal" href="../pep-0304/" title="PEP 304 – Controlling Generation of Bytecode Files">PEP 304</a> does not indicate
|
||
why it was withdrawn, shadow directories have a number of problems.
|
||
The location of the shadow <code class="docutils literal notranslate"><span class="pre">pyc</span></code> files would not be easily discovered
|
||
and would depend on the proper and consistent use of the
|
||
<code class="docutils literal notranslate"><span class="pre">$PYTHONBYTECODE</span></code> environment variable both by the system and by end
|
||
users. There are also global implications, meaning that while the
|
||
system might want to shadow <code class="docutils literal notranslate"><span class="pre">pyc</span></code> files, users might not want to, but
|
||
the PEP defines only an all-or-nothing approach.</p>
|
||
<p>As an example of the problem, a common (though fragile) Python idiom
|
||
for locating data files is to do something like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">os</span> <span class="kn">import</span> <span class="n">dirname</span><span class="p">,</span> <span class="n">join</span>
|
||
<span class="kn">import</span> <span class="nn">foo.bar</span>
|
||
<span class="n">data_file</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">dirname</span><span class="p">(</span><span class="n">foo</span><span class="o">.</span><span class="n">bar</span><span class="o">.</span><span class="vm">__file__</span><span class="p">),</span> <span class="s1">'my.dat'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This would be problematic since <code class="docutils literal notranslate"><span class="pre">foo.bar.__file__</span></code> will give the
|
||
location of the <code class="docutils literal notranslate"><span class="pre">pyc</span></code> file in the shadow directory, and it may not be
|
||
possible to find the <code class="docutils literal notranslate"><span class="pre">my.dat</span></code> file relative to the source directory
|
||
from there.</p>
|
||
</section>
|
||
<section id="fat-byte-compilation-files">
|
||
<h3><a class="toc-backref" href="#fat-byte-compilation-files" role="doc-backlink">Fat byte compilation files</a></h3>
|
||
<p>An earlier version of this PEP described “fat” Python byte code files.
|
||
These files would contain the equivalent of multiple <code class="docutils literal notranslate"><span class="pre">pyc</span></code> files in a
|
||
single <code class="docutils literal notranslate"><span class="pre">pyf</span></code> file, with a lookup table keyed off the appropriate magic
|
||
number. This was an extensible file format so that the first 5
|
||
parallel Python implementations could be supported fairly efficiently,
|
||
but with extension lookup tables available to scale <code class="docutils literal notranslate"><span class="pre">pyf</span></code> byte code
|
||
objects as large as necessary.</p>
|
||
<p>The fat byte compilation files were fairly complex, and inherently
|
||
introduced difficult race conditions, so the current simplification of
|
||
using directories was suggested. The same problem applies to using
|
||
zip files as the fat pyc file format.</p>
|
||
</section>
|
||
<section id="multiple-file-extensions">
|
||
<h3><a class="toc-backref" href="#multiple-file-extensions" role="doc-backlink">Multiple file extensions</a></h3>
|
||
<p>The PEP author also considered an approach where multiple thin byte
|
||
compiled files lived in the same place, but used different file
|
||
extensions to designate the Python version. E.g. foo.pyc25,
|
||
foo.pyc26, foo.pyc31 etc. This was rejected because of the clutter
|
||
involved in writing so many different files. The multiple extension
|
||
approach makes it more difficult (and an ongoing task) to update any
|
||
tools that are dependent on the file extension.</p>
|
||
</section>
|
||
<section id="pyc">
|
||
<h3><a class="toc-backref" href="#pyc" role="doc-backlink">.pyc</a></h3>
|
||
<p>A proposal was floated to call the <code class="docutils literal notranslate"><span class="pre">__pycache__</span></code> directory <code class="docutils literal notranslate"><span class="pre">.pyc</span></code> or
|
||
some other dot-file name. This would have the effect on *nix systems
|
||
of hiding the directory. There are many reasons why this was
|
||
rejected by the BDFL <a class="footnote-reference brackets" href="#id37" id="id18">[20]</a> including the fact that dot-files are only
|
||
special on some platforms, and we actually do <em>not</em> want to hide these
|
||
completely from users.</p>
|
||
</section>
|
||
</section>
|
||
<section id="reference-implementation">
|
||
<h2><a class="toc-backref" href="#reference-implementation" role="doc-backlink">Reference implementation</a></h2>
|
||
<p>Work on this code is tracked in a Bazaar branch on Launchpad <a class="footnote-reference brackets" href="#id38" id="id19">[22]</a>
|
||
until it’s ready for merge into Python 3.2. The work-in-progress diff
|
||
can also be viewed <a class="footnote-reference brackets" href="#id39" id="id20">[23]</a> and is updated automatically as new changes
|
||
are uploaded.</p>
|
||
<p>A Rietveld code review issue <a class="footnote-reference brackets" href="#id40" id="id21">[24]</a> has been opened as of 2010-04-01 (no,
|
||
this is not an April Fools joke :).</p>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id22" role="doc-footnote">
|
||
<dt class="label" id="id22">[<a href="#id1">2</a>]</dt>
|
||
<dd>The marshal module:
|
||
<a class="reference external" href="https://docs.python.org/3.1/library/marshal.html">https://docs.python.org/3.1/library/marshal.html</a></aside>
|
||
<aside class="footnote brackets" id="id23" role="doc-footnote">
|
||
<dt class="label" id="id23">[<a href="#id2">3</a>]</dt>
|
||
<dd>import.c:
|
||
<a class="reference external" href="https://github.com/python/cpython/blob/v3.2a1/Python/import.c">https://github.com/python/cpython/blob/v3.2a1/Python/import.c</a></aside>
|
||
<aside class="footnote brackets" id="id24" role="doc-footnote">
|
||
<dt class="label" id="id24">[<a href="#id3">4</a>]</dt>
|
||
<dd>Ubuntu: <a class="reference external" href="https://www.ubuntu.com">https://www.ubuntu.com</a></aside>
|
||
<aside class="footnote brackets" id="id25" role="doc-footnote">
|
||
<dt class="label" id="id25">[<a href="#id4">5</a>]</dt>
|
||
<dd>Debian: <a class="reference external" href="https://www.debian.org">https://www.debian.org</a></aside>
|
||
<aside class="footnote brackets" id="id26" role="doc-footnote">
|
||
<dt class="label" id="id26">[<a href="#id5">6</a>]</dt>
|
||
<dd>Debian Python Policy:
|
||
<a class="reference external" href="https://www.debian.org/doc/packaging-manuals/python-policy/">https://www.debian.org/doc/packaging-manuals/python-policy/</a></aside>
|
||
<aside class="footnote brackets" id="id27" role="doc-footnote">
|
||
<dt class="label" id="id27">[<a href="#id6">8</a>]</dt>
|
||
<dd>python-support:
|
||
<a class="reference external" href="https://web.archive.org/web/20100110123824/http://wiki.debian.org/DebianPythonFAQ#Whatispython-support.3F">https://web.archive.org/web/20100110123824/http://wiki.debian.org/DebianPythonFAQ#Whatispython-support.3F</a></aside>
|
||
<aside class="footnote brackets" id="id28" role="doc-footnote">
|
||
<dt class="label" id="id28">[<a href="#id7">9</a>]</dt>
|
||
<dd>python-central:
|
||
<a class="reference external" href="https://web.archive.org/web/20100110123824/http://wiki.debian.org/DebianPythonFAQ#Whatispython-central.3F">https://web.archive.org/web/20100110123824/http://wiki.debian.org/DebianPythonFAQ#Whatispython-central.3F</a></aside>
|
||
<aside class="footnote brackets" id="id29" role="doc-footnote">
|
||
<dt class="label" id="id29">[<a href="#id17">10</a>]</dt>
|
||
<dd>binascii.hexlify():
|
||
<a class="reference external" href="https://docs.python.org/3.1/library/binascii.html#binascii.hexlify">https://docs.python.org/3.1/library/binascii.html#binascii.hexlify</a></aside>
|
||
<aside class="footnote brackets" id="id30" role="doc-footnote">
|
||
<dt class="label" id="id30">[<a href="#id10">11</a>]</dt>
|
||
<dd>Jython: <a class="reference external" href="http://www.jython.org/">http://www.jython.org/</a></aside>
|
||
<aside class="footnote brackets" id="id31" role="doc-footnote">
|
||
<dt class="label" id="id31">[<a href="#id11">12</a>]</dt>
|
||
<dd>IronPython: <a class="reference external" href="http://ironpython.net/">http://ironpython.net/</a></aside>
|
||
<aside class="footnote brackets" id="id32" role="doc-footnote">
|
||
<dt class="label" id="id32">[<a href="#id12">13</a>]</dt>
|
||
<dd>PyPy: <a class="reference external" href="https://web.archive.org/web/20100310130136/http://codespeak.net/pypy/dist/pypy/doc/">https://web.archive.org/web/20100310130136/http://codespeak.net/pypy/dist/pypy/doc/</a></aside>
|
||
<aside class="footnote brackets" id="id33" role="doc-footnote">
|
||
<dt class="label" id="id33">[<a href="#id13">14</a>]</dt>
|
||
<dd>Pynie: <a class="reference external" href="https://code.google.com/archive/p/pynie/">https://code.google.com/archive/p/pynie/</a></aside>
|
||
<aside class="footnote brackets" id="id34" role="doc-footnote">
|
||
<dt class="label" id="id34">[15]<em> (<a href='#id8'>1</a>, <a href='#id14'>2</a>) </em></dt>
|
||
<dd>py_compile: <a class="reference external" href="https://docs.python.org/3.1/library/py_compile.html">https://docs.python.org/3.1/library/py_compile.html</a></aside>
|
||
<aside class="footnote brackets" id="id35" role="doc-footnote">
|
||
<dt class="label" id="id35">[16]<em> (<a href='#id9'>1</a>, <a href='#id15'>2</a>) </em></dt>
|
||
<dd>compileall: <a class="reference external" href="https://docs.python.org/3.1/library/compileall.html">https://docs.python.org/3.1/library/compileall.html</a></aside>
|
||
<aside class="footnote brackets" id="id36" role="doc-footnote">
|
||
<dt class="label" id="id36">[<a href="#id16">17</a>]</dt>
|
||
<dd>imp: <a class="reference external" href="https://docs.python.org/3.1/library/imp.html">https://docs.python.org/3.1/library/imp.html</a></aside>
|
||
<aside class="footnote brackets" id="id37" role="doc-footnote">
|
||
<dt class="label" id="id37">[<a href="#id18">20</a>]</dt>
|
||
<dd><a class="reference external" href="https://www.mail-archive.com/python-dev@python.org/msg45203.html">https://www.mail-archive.com/python-dev@python.org/msg45203.html</a></aside>
|
||
</aside>
|
||
<p>[21] importlib: <a class="reference external" href="https://docs.python.org/3.1/library/importlib.html">https://docs.python.org/3.1/library/importlib.html</a></p>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id38" role="doc-footnote">
|
||
<dt class="label" id="id38">[<a href="#id19">22</a>]</dt>
|
||
<dd><a class="reference external" href="https://code.launchpad.net/~barry/python/pep3147">https://code.launchpad.net/~barry/python/pep3147</a></aside>
|
||
<aside class="footnote brackets" id="id39" role="doc-footnote">
|
||
<dt class="label" id="id39">[<a href="#id20">23</a>]</dt>
|
||
<dd><a class="reference external" href="https://code.launchpad.net/~barry/python/pep3147/+merge/22648">https://code.launchpad.net/~barry/python/pep3147/+merge/22648</a></aside>
|
||
<aside class="footnote brackets" id="id40" role="doc-footnote">
|
||
<dt class="label" id="id40">[<a href="#id21">24</a>]</dt>
|
||
<dd><a class="reference external" href="http://codereview.appspot.com/842043/show">http://codereview.appspot.com/842043/show</a></aside>
|
||
</aside>
|
||
</section>
|
||
<section id="acknowledgments">
|
||
<h2><a class="toc-backref" href="#acknowledgments" role="doc-backlink">ACKNOWLEDGMENTS</a></h2>
|
||
<p>Barry Warsaw’s original idea was for fat Python byte code files.
|
||
Martin von Loewis reviewed an early draft of the PEP and suggested the
|
||
simplification to store traditional <code class="docutils literal notranslate"><span class="pre">pyc</span></code> and <code class="docutils literal notranslate"><span class="pre">pyo</span></code> files in a
|
||
directory. Many other people reviewed early versions of this PEP and
|
||
provided useful feedback including but not limited to:</p>
|
||
<ul class="simple">
|
||
<li>David Malcolm</li>
|
||
<li>Josselin Mouette</li>
|
||
<li>Matthias Klose</li>
|
||
<li>Michael Hudson</li>
|
||
<li>Michael Vogt</li>
|
||
<li>Piotr Ożarowski</li>
|
||
<li>Scott Kitterman</li>
|
||
<li>Toshio Kuratomi</li>
|
||
</ul>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-3147.rst">https://github.com/python/peps/blob/main/peps/pep-3147.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-3147.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#background">Background</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a><ul>
|
||
<li><a class="reference internal" href="#examples">Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#python-behavior">Python behavior</a><ul>
|
||
<li><a class="reference internal" href="#case-0-the-steady-state">Case 0: The steady state</a></li>
|
||
<li><a class="reference internal" href="#case-1-the-first-import">Case 1: The first import</a></li>
|
||
<li><a class="reference internal" href="#case-2-the-second-import">Case 2: The second import</a></li>
|
||
<li><a class="reference internal" href="#case-3-pycache-foo-magic-pyc-with-no-source">Case 3: __pycache__/foo.<magic>.pyc with no source</a></li>
|
||
<li><a class="reference internal" href="#case-4-legacy-pyc-files-and-source-less-imports">Case 4: legacy pyc files and source-less imports</a></li>
|
||
<li><a class="reference internal" href="#case-5-read-only-file-systems">Case 5: read-only file systems</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#flow-chart">Flow chart</a></li>
|
||
<li><a class="reference internal" href="#alternative-python-implementations">Alternative Python implementations</a></li>
|
||
<li><a class="reference internal" href="#implementation-strategy">Implementation strategy</a></li>
|
||
<li><a class="reference internal" href="#effects-on-existing-code">Effects on existing code</a><ul>
|
||
<li><a class="reference internal" href="#detecting-pep-3147-availability">Detecting PEP 3147 availability</a></li>
|
||
<li><a class="reference internal" href="#file">__file__</a></li>
|
||
<li><a class="reference internal" href="#py-compile-and-compileall">py_compile and compileall</a></li>
|
||
<li><a class="reference internal" href="#bdist-wininst-and-the-windows-installer">bdist_wininst and the Windows installer</a></li>
|
||
<li><a class="reference internal" href="#file-extension-checks">File extension checks</a></li>
|
||
<li><a class="reference internal" href="#backports">Backports</a></li>
|
||
<li><a class="reference internal" href="#makefiles-and-other-dependency-tools">Makefiles and other dependency tools</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#alternatives">Alternatives</a><ul>
|
||
<li><a class="reference internal" href="#hexadecimal-magic-tags">Hexadecimal magic tags</a></li>
|
||
<li><a class="reference internal" href="#pep-304">PEP 304</a></li>
|
||
<li><a class="reference internal" href="#fat-byte-compilation-files">Fat byte compilation files</a></li>
|
||
<li><a class="reference internal" href="#multiple-file-extensions">Multiple file extensions</a></li>
|
||
<li><a class="reference internal" href="#pyc">.pyc</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#acknowledgments">ACKNOWLEDGMENTS</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-3147.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |