mirror of https://github.com/python/peps
1161 lines
96 KiB
HTML
1161 lines
96 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 668 – Marking Python base environments as “externally managed” | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0668/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 668 – Marking Python base environments as “externally managed” | peps.python.org'>
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0668/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Python Enhancement Proposals (PEPs)">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 668</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 668 – Marking Python base environments as “externally managed”</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Geoffrey Thomas <geofft at ldpreload.com>,
|
||
Matthias Klose <doko at ubuntu.com>,
|
||
Filipe Laíns <lains at riseup.net>,
|
||
Donald Stufft <donald at stufft.io>,
|
||
Tzu-ping Chung <uranusjr at gmail.com>,
|
||
Stefano Rivera <stefanor at debian.org>,
|
||
Elana Hashman <ehashman at debian.org>,
|
||
Pradyun Gedam <pradyunsg at gmail.com></dd>
|
||
<dt class="field-even">PEP-Delegate<span class="colon">:</span></dt>
|
||
<dd class="field-even">Paul Moore <p.f.moore at gmail.com></dd>
|
||
<dt class="field-odd">Discussions-To<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://discuss.python.org/t/10302">Discourse thread</a></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Normative proposal accepted for implementation">Accepted</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Topic<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="../topic/packaging/">Packaging</a></dd>
|
||
<dt class="field-odd">Created<span class="colon">:</span></dt>
|
||
<dd class="field-odd">18-May-2021</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">28-May-2021</dd>
|
||
<dt class="field-odd">Resolution<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://discuss.python.org/t/10302/44">Discourse message</a></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#terminology">Terminology</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a><ul>
|
||
<li><a class="reference internal" href="#use-cases">Use cases</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#marking-an-interpreter-as-using-an-external-package-manager">Marking an interpreter as using an external package manager</a></li>
|
||
<li><a class="reference internal" href="#writing-to-only-the-target-sysconfig-scheme">Writing to only the target <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#recommendations-for-distros">Recommendations for distros</a><ul>
|
||
<li><a class="reference internal" href="#mark-the-installation-as-externally-managed">Mark the installation as externally managed</a></li>
|
||
<li><a class="reference internal" href="#guide-users-towards-virtual-environments">Guide users towards virtual environments</a></li>
|
||
<li><a class="reference internal" href="#keep-the-marker-file-in-container-images">Keep the marker file in container images</a></li>
|
||
<li><a class="reference internal" href="#create-separate-distro-and-local-directories">Create separate distro and local directories</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#security-implications">Security Implications</a></li>
|
||
<li><a class="reference internal" href="#alternatives">Alternatives</a><ul>
|
||
<li><a class="reference internal" href="#marker-file">Marker file</a></li>
|
||
<li><a class="reference internal" href="#system-python">System Python</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#implementation-notes">Implementation Notes</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<div class="pep-banner canonical-pypa-spec sticky-banner admonition attention">
|
||
<p class="admonition-title">Attention</p>
|
||
<p>This PEP is a historical document. The up-to-date, canonical spec, <a class="reference external" href="https://packaging.python.org/en/latest/specifications/externally-managed-environments/#externally-managed-environments" title="(in Python Packaging User Guide)"><span>Externally Managed Environments</span></a>, is maintained on the <a class="reference external" href="https://packaging.python.org/en/latest/specifications/">PyPA specs page</a>.</p>
|
||
<p class="close-button">×</p>
|
||
<p>See the <a class="reference external" href="https://www.pypa.io/en/latest/specifications/#handling-fixes-and-other-minor-updates">PyPA specification update process</a> for how to propose changes.</p>
|
||
</div>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>A long-standing practical problem for Python users has been conflicts
|
||
between OS package managers and Python-specific package management
|
||
tools like pip. These conflicts include both Python-level API
|
||
incompatibilities and conflicts over file ownership.</p>
|
||
<p>Historically, Python-specific package management tools have defaulted
|
||
to installing packages into an implicit global context. With the
|
||
standardization and popularity of virtual environments, a better
|
||
solution for most (but not all) use cases is to use Python-specific
|
||
package management tools only within a virtual environment.</p>
|
||
<p>This PEP proposes a mechanism for a Python installation to communicate
|
||
to tools like pip that its global package installation context is
|
||
managed by some means external to Python, such as an OS package
|
||
manager. It specifies that Python-specific package management tools
|
||
should neither install nor remove packages into the interpreter’s
|
||
global context, by default, and should instead guide the end user
|
||
towards using a virtual environment.</p>
|
||
<p>It also standardizes an interpretation of the <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> schemes so
|
||
that, if a Python-specific package manager is about to install a
|
||
package in an interpreter-wide context, it can do so in a manner that
|
||
will avoid conflicting with the external package manager and reduces
|
||
the risk of breaking software shipped by the external package manager.</p>
|
||
</section>
|
||
<section id="terminology">
|
||
<h2><a class="toc-backref" href="#terminology" role="doc-backlink">Terminology</a></h2>
|
||
<p>A few terms used in this PEP have multiple meanings in the contexts
|
||
that it spans. For clarity, this PEP uses the following terms in
|
||
specific ways:</p>
|
||
<dl>
|
||
<dt>distro</dt><dd>Short for “distribution,” a collection of various sorts of
|
||
software, ideally designed to work properly together, including
|
||
(in contexts relevant to this document) the Python interpreter
|
||
itself, software written in Python, and software written in other
|
||
languages. That is, this is the sense used in phrases such as
|
||
“Linux distro” or “Berkeley Software Distribution.”<p>A distro can be an operating system (OS) of its own, such as
|
||
Debian, Fedora, or FreeBSD. It can also be an overlay distribution
|
||
that installs on top of an existing OS, such as Homebrew or
|
||
MacPorts.</p>
|
||
<p>This document uses the short term “distro,” because the term
|
||
“distribution” has another meaning in Python packaging contexts: a
|
||
source or binary distribution package of a single piece of Python
|
||
language software, that is, in the sense of
|
||
<code class="docutils literal notranslate"><span class="pre">setuptools.dist.Distribution</span></code> or “sdist”. To avoid confusion,
|
||
this document does not use the plain term “distribution” at all.
|
||
In the Python packaging sense, it uses the full phrase
|
||
“distribution package” or just “package” (see below).</p>
|
||
<p>The provider of a distro - the team or company that collects and
|
||
publishes the software and makes any needed modifications - is its
|
||
<strong>distributor</strong>.</p>
|
||
</dd>
|
||
<dt>package</dt><dd>A unit of software that can be installed and used within Python.
|
||
That is, this refers to what Python-specific packaging tools tend
|
||
to call a “<a class="reference external" href="https://packaging.python.org/glossary/#term-Distribution-Package">distribution package</a>” or simply a “distribution”;
|
||
the colloquial abbreviation “package” is used in the sense of the
|
||
Python Package Index.<p>This document does not use “package” in the sense of an importable
|
||
name that contains Python modules, though in many cases, a
|
||
distribution package consists of a single importable package of
|
||
the same name.</p>
|
||
<p>This document generally does not use the term “package” to refer
|
||
to units of installation by a distro’s package manager (such as
|
||
<code class="docutils literal notranslate"><span class="pre">.deb</span></code> or <code class="docutils literal notranslate"><span class="pre">.rpm</span></code> files). When needed, it uses phrasing such as
|
||
“a distro’s package.” (Again, in many cases, a Python package is
|
||
shipped inside a distro’s package named something like <code class="docutils literal notranslate"><span class="pre">python-</span></code>
|
||
plus the Python package name.)</p>
|
||
</dd>
|
||
<dt>Python-specific package manager</dt><dd>A tool for installing, upgrading, and/or removing Python packages
|
||
in a manner that conforms to Python packaging standards (such as
|
||
<a class="pep reference internal" href="../pep-0376/" title="PEP 376 – Database of Installed Python Distributions">PEP 376</a> and <a class="pep reference internal" href="../pep-0427/" title="PEP 427 – The Wheel Binary Package Format 1.0">PEP 427</a>). The most popular Python-specific package
|
||
manager is pip <a class="footnote-reference brackets" href="#pip" id="id1">[1]</a>; other examples include the old Easy
|
||
Install command <a class="footnote-reference brackets" href="#easy-install" id="id2">[2]</a> as well as direct usage of a
|
||
<code class="docutils literal notranslate"><span class="pre">setup.py</span></code> command.<p>(Conda <a class="footnote-reference brackets" href="#conda" id="id3">[3]</a> is a bit of a special case, as the <code class="docutils literal notranslate"><span class="pre">conda</span></code>
|
||
command can install much more than just Python packages, making it
|
||
more like a distro package manager in some senses. Since the
|
||
<code class="docutils literal notranslate"><span class="pre">conda</span></code> command generally only operates on Conda-created
|
||
environments, most of the concerns in this document do not apply
|
||
to <code class="docutils literal notranslate"><span class="pre">conda</span></code> when acting as a Python-specific package manager.)</p>
|
||
</dd>
|
||
<dt>distro package manager</dt><dd>A tool for installing, upgrading, and/or removing a distro’s
|
||
packages in an installed instance of that distro, which is capable
|
||
of installing Python packages as well as non-Python packages, and
|
||
therefore generally has its own database of installed software
|
||
unrelated to <a class="pep reference internal" href="../pep-0376/" title="PEP 376 – Database of Installed Python Distributions">PEP 376</a>. Examples include <code class="docutils literal notranslate"><span class="pre">apt</span></code>, <code class="docutils literal notranslate"><span class="pre">dpkg</span></code>, <code class="docutils literal notranslate"><span class="pre">dnf</span></code>,
|
||
<code class="docutils literal notranslate"><span class="pre">rpm</span></code>, <code class="docutils literal notranslate"><span class="pre">pacman</span></code>, and <code class="docutils literal notranslate"><span class="pre">brew</span></code>. The salient feature is that if
|
||
a package was installed by a distro package manager, removing or
|
||
upgrading it in a way that would satisfy a Python-specific package
|
||
manager will generally leave a distro package manager in an
|
||
inconsistent state.<p>This document also uses phrases like “external package manager” or
|
||
“system’s package manager” to refer to a distro package manager in
|
||
certain contexts.</p>
|
||
</dd>
|
||
<dt>shadow</dt><dd>To shadow an installed Python package is to cause some other
|
||
package to be preferred for imports without removing any files
|
||
from the shadowed package. This requires multiple entries on
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>: if package A 2.0 installs module <code class="docutils literal notranslate"><span class="pre">a.py</span></code> in one
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code> entry, and package A 1.0 installs module <code class="docutils literal notranslate"><span class="pre">a.py</span></code> in
|
||
a later <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> entry, then <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">a</span></code> returns the module
|
||
from the former, and we say that A 2.0 shadows A 1.0.</dd>
|
||
</dl>
|
||
</section>
|
||
<section id="motivation">
|
||
<h2><a class="toc-backref" href="#motivation" role="doc-backlink">Motivation</a></h2>
|
||
<p>Thanks to Python’s immense popularity, software distros (by which we
|
||
mean Linux and other OS distros as well as overlay distros like
|
||
Homebrew and MacPorts) generally ship Python for two purposes: as a
|
||
software package to be used in its own right by end users, and as a
|
||
language dependency for other software in the distro.</p>
|
||
<p>For example, Fedora and Debian (and their downstream distros, as well
|
||
as many others) ship a <code class="docutils literal notranslate"><span class="pre">/usr/bin/python3</span></code> binary which provides the
|
||
<code class="docutils literal notranslate"><span class="pre">python3</span></code> command available to end users as well as the
|
||
<code class="docutils literal notranslate"><span class="pre">#!/usr/bin/python3</span></code> shebang for Python-language software included
|
||
in the distro. Because there are no official binary releases of Python
|
||
for Linux/UNIX, almost all Python end users on these OSes use the
|
||
Python interpreter built and shipped with their distro.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">python3</span></code> executable available to the users of the distro and
|
||
the <code class="docutils literal notranslate"><span class="pre">python3</span></code> executable available as a dependency for other
|
||
software in the distro are typically the same binary. This means that
|
||
if an end user installs a Python package using a tool like <code class="docutils literal notranslate"><span class="pre">pip</span></code>
|
||
outside the context of a virtual environment, that package is visible
|
||
to Python-language software shipped by the distro. If the
|
||
newly-installed package (or one of its dependencies) is a newer,
|
||
backwards-incompatible version of a package that was installed through
|
||
the distro, it may break software shipped by the distro.</p>
|
||
<p>This may pose a critical problem for the integrity of distros, which
|
||
often have package-management tools that are themselves written in
|
||
Python. For example, it’s possible to unintentionally break Fedora’s
|
||
<code class="docutils literal notranslate"><span class="pre">dnf</span></code> command with a <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> command, making it hard to
|
||
recover.</p>
|
||
<p>This applies both to system-wide installs (<code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span> <span class="pre">install</span></code>) as
|
||
well as user home directory installs (<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code>), since
|
||
packages in either location show up on the <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> of
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/bin/python3</span></code>.</p>
|
||
<p>There is a worse problem with system-wide installs: if you attempt to
|
||
recover from this situation with <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span> <span class="pre">uninstall</span></code>, you may end
|
||
up removing packages that are shipped by the system’s package manager.
|
||
In fact, this can even happen if you simply upgrade a package - pip
|
||
will try to remove the old version of the package, as shipped by the
|
||
OS. At this point it may not be possible to recover the system to a
|
||
consistent state using just the software remaining on the system.</p>
|
||
<p>Over the past many years, a consensus has emerged that the best way to
|
||
install Python libraries or applications (when not using a distro’s
|
||
package) is to use a virtual environment. This approach was
|
||
popularized by the PyPA <a class="reference external" href="https://virtualenv.pypa.io/en/latest/">virtualenv</a> project, and a simple version of
|
||
that approach is now available in the Python standard library as
|
||
<code class="docutils literal notranslate"><span class="pre">venv</span></code>. Installing a Python package into a virtualenv prevents it
|
||
from being visible to the unqualified <code class="docutils literal notranslate"><span class="pre">/usr/bin/python3</span></code> interpreter
|
||
and prevents breaking system software.</p>
|
||
<p>In some cases, however, it’s useful and intentional to install a
|
||
Python package from outside of the distro that influences the behavior
|
||
of distro-shipped commands. This is common in the case of software
|
||
like Sphinx or Ansible which have a mechanism for writing
|
||
Python-language extensions. A user may want to use their distro’s
|
||
version of the base software (for reasons of paid support or security
|
||
updates) but install a small extension from PyPI, and they’d want that
|
||
extension to be importable by the software in their base system.</p>
|
||
<p>While this continues to carry the risk of installing a newer version
|
||
of a dependency than the operating system expects or otherwise
|
||
negatively affecting the behavior of an application, it does not need
|
||
to carry the risk of removing files from the operating system. A tool
|
||
like pip should be able to install packages in some directory on the
|
||
default <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, if specifically requested, without deleting
|
||
files owned by the system’s package manager.</p>
|
||
<p>Therefore, this PEP proposes two things.</p>
|
||
<p>First, it proposes <strong>a way for distributors of a Python interpreter to
|
||
mark that interpreter as having its packages managed by means external
|
||
to Python</strong>, such that Python-specific tools like pip should not
|
||
change the installed packages in the interpreter’s global <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>
|
||
in any way (add, upgrade/downgrade, or remove) unless specifically
|
||
overridden. It also provides a means for the distributor to indicate
|
||
how to use a virtual environment as an alternative.</p>
|
||
<p>This is an opt-in mechanism: by default, the Python interpreter
|
||
compiled from upstream sources will not be so marked, and so running
|
||
<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> with a self-compiled interpreter, or with a distro
|
||
that has not explicitly marked its interpreter, will work as it always
|
||
has worked.</p>
|
||
<p>Second, it sets the rule that when installing packages to an
|
||
interpreter’s global context (either to an unmarked interpreter, or if
|
||
overriding the marking), <strong>Python-specific package managers should
|
||
modify or delete files only within the directories of the sysconfig
|
||
scheme in which they would create files</strong>. This permits a distributor
|
||
of a Python interpreter to set up two directories, one for its own
|
||
managed packages, and one for unmanaged packages installed by the end
|
||
user, and ensure that installing unmanaged packages will not delete
|
||
(or overwrite) files owned by the external package manager.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>As described in detail in the next section, the first behavior change
|
||
involves creating a marker file named <code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code>, whose
|
||
presence indicates that non-virtual-environment package installations
|
||
are managed by some means external to Python, such as a distro’s
|
||
package manager. This file is specified to live in the <code class="docutils literal notranslate"><span class="pre">stdlib</span></code>
|
||
directory in the default <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme, which marks the
|
||
interpreter / installation as a whole, not a particular location on
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>. The reason for this is that, as identified above, there
|
||
are two related problems that risk breaking an externally-managed
|
||
Python: you can install an incompatible new version of a package
|
||
system-wide (e.g., with <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span> <span class="pre">install</span></code>), and you can install one
|
||
in your user account alone, but in a location that is on the standard
|
||
Python command’s <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> (e.g., with <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code>). If
|
||
the marker file were in the system-wide <code class="docutils literal notranslate"><span class="pre">site-packages</span></code> directory,
|
||
it would not clearly apply to the second case. The <a class="reference internal" href="#alternatives">Alternatives</a>
|
||
section has further discussion of possible locations.</p>
|
||
<p>The second behavior change takes advantage of the existing
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> setup in distros that have already encountered this
|
||
class of problem, and specifically addresses the problem of a
|
||
Python-specific package manager deleting or overwriting files that are
|
||
owned by an external package manager.</p>
|
||
<section id="use-cases">
|
||
<h3><a class="toc-backref" href="#use-cases" role="doc-backlink">Use cases</a></h3>
|
||
<p>The changed behavior in this PEP is intended to “do the right thing”
|
||
for as many use cases as possible. In this section, we consider the
|
||
changes specified by this PEP for several representative use cases /
|
||
contexts. Specifically, we ask about the two behaviors that could be
|
||
changed by this PEP:</p>
|
||
<ol class="arabic simple">
|
||
<li>Will a Python-specific installer tool like <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> permit
|
||
installations by default, after implementation of this PEP?</li>
|
||
<li>If you do run such a tool, should it be willing to delete packages
|
||
shipped by the external (non-Python-specific) package manager for
|
||
that context, such as a distro package manager?</li>
|
||
</ol>
|
||
<p>(For simplicity, this section discusses pip as the Python-specific
|
||
installer tool, though the analysis should apply equally to any other
|
||
Python-specific package management tool.)</p>
|
||
<p>This table summarizes the use cases discussed in detail below:</p>
|
||
<table class="docutils align-default">
|
||
<thead>
|
||
<tr class="row-odd"><th class="head">Case</th>
|
||
<th class="head">Description</th>
|
||
<th class="head"><code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> permitted</th>
|
||
<th class="head">Deleting externally-installed packages permitted</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td>1</td>
|
||
<td>Unpatched CPython</td>
|
||
<td>Currently yes; stays yes</td>
|
||
<td>Currently yes; stays yes</td>
|
||
</tr>
|
||
<tr class="row-odd"><td>2</td>
|
||
<td>Distro <code class="docutils literal notranslate"><span class="pre">/usr/bin/python3</span></code></td>
|
||
<td>Currently yes; becomes no
|
||
(assuming the distro
|
||
adds a marker file)</td>
|
||
<td>Currently yes (except on Debian); becomes no</td>
|
||
</tr>
|
||
<tr class="row-even"><td>3</td>
|
||
<td>Distro Python in venv</td>
|
||
<td>Currently yes; stays yes</td>
|
||
<td>There are no externally-installed packages</td>
|
||
</tr>
|
||
<tr class="row-odd"><td>4</td>
|
||
<td>Distro Python in venv
|
||
with <code class="docutils literal notranslate"><span class="pre">--system-site-packages</span></code></td>
|
||
<td>Currently yes; stays yes</td>
|
||
<td>Currently no; stays no</td>
|
||
</tr>
|
||
<tr class="row-even"><td>5</td>
|
||
<td>Distro Python in Docker</td>
|
||
<td>Currently yes; becomes no
|
||
(assuming the distro
|
||
adds a marker file)</td>
|
||
<td>Currently yes; becomes no</td>
|
||
</tr>
|
||
<tr class="row-odd"><td>6</td>
|
||
<td>Conda environment</td>
|
||
<td>Currently yes; stays yes</td>
|
||
<td>Currently yes; stays yes</td>
|
||
</tr>
|
||
<tr class="row-even"><td>7</td>
|
||
<td>Dev-facing distro</td>
|
||
<td>Currently yes; becomes no
|
||
(assuming they add a
|
||
marker file)</td>
|
||
<td>Currently often yes; becomes no
|
||
(assuming they configure <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> as needed)</td>
|
||
</tr>
|
||
<tr class="row-odd"><td>8</td>
|
||
<td>Distro building packages</td>
|
||
<td>Currently yes; can stay yes</td>
|
||
<td>Currently yes; becomes no</td>
|
||
</tr>
|
||
<tr class="row-even"><td>9</td>
|
||
<td><code class="docutils literal notranslate"><span class="pre">PYTHONHOME</span></code> copied from
|
||
a distro Python stdlib</td>
|
||
<td>Currently yes; becomes no</td>
|
||
<td>Currently yes; becomes no</td>
|
||
</tr>
|
||
<tr class="row-odd"><td>10</td>
|
||
<td><code class="docutils literal notranslate"><span class="pre">PYTHONHOME</span></code> copied from
|
||
upstream Python stdlib</td>
|
||
<td>Currently yes; stays yes</td>
|
||
<td>Currently yes; stays yes</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>In more detail, the use cases above are:</p>
|
||
<ol class="arabic">
|
||
<li>A standard unpatched CPython, without any special configuration of
|
||
or patches to <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> and without a marker file. This PEP
|
||
does not change its behavior.<p>Such a CPython should (regardless of this PEP) not be installed in
|
||
a way that overlaps any distro-installed Python on the same system.
|
||
For instance, on an OS that ships Python in <code class="docutils literal notranslate"><span class="pre">/usr/bin</span></code>, you
|
||
should not install a custom CPython built with <code class="docutils literal notranslate"><span class="pre">./configure</span>
|
||
<span class="pre">--prefix=/usr</span></code>, or it will overwrite some files from the distro
|
||
and the distro will eventually overwrite some files from your
|
||
installation. Instead, your installation should be in a separate
|
||
directory (perhaps <code class="docutils literal notranslate"><span class="pre">/usr/local</span></code>, <code class="docutils literal notranslate"><span class="pre">/opt</span></code>, or your home
|
||
directory).</p>
|
||
<p>Therefore, we can assume that such a CPython has its own <code class="docutils literal notranslate"><span class="pre">stdlib</span></code>
|
||
directory and its own <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> schemes that do not overlap any
|
||
distro-installed Python. So any OS-installed packages are not
|
||
visible or relevant here.</p>
|
||
<p>If there is a concept of “externally-installed” packages in this
|
||
case, it’s something outside the OS and generally managed by
|
||
whoever built and installed this CPython. Because the installer
|
||
chose not to add a marker file or modify <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> schemes,
|
||
they’re choosing the current behavior, and <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> can
|
||
remove any packages available in this CPython.</p>
|
||
</li>
|
||
<li>A distro’s <code class="docutils literal notranslate"><span class="pre">/usr/bin/python3</span></code>, either when running <code class="docutils literal notranslate"><span class="pre">pip</span>
|
||
<span class="pre">install</span></code> as root or <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code>, following our
|
||
<a class="reference internal" href="#recommendations-for-distros">Recommendations for distros</a>.<p>These recommendations include shipping a marker file in the
|
||
<code class="docutils literal notranslate"><span class="pre">stdlib</span></code> directory, to prevent <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> by default, and
|
||
placing distro-shipped packages in a location other than the
|
||
default <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme, so that <code class="docutils literal notranslate"><span class="pre">pip</span></code> as root does not
|
||
write to that location.</p>
|
||
<p>Many distros (including Debian, Fedora, and their derivatives) are
|
||
already doing the latter.</p>
|
||
<p>On Debian and derivatives, <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> does not currently
|
||
delete distro-installed packages, because Debian carries a <a class="reference external" href="https://sources.debian.org/src/python-pip/20.3.4-2/debian/patches/hands-off-system-packages.patch/">patch
|
||
to pip to prevent this</a>. So, for those distros, this PEP is not a
|
||
behavior change; it simply standardizes that behavior in a way that
|
||
is no longer Debian-specific and can be included into upstream pip.</p>
|
||
<p>(We have seen user reports of externally-installed packages being
|
||
deleted on Debian or a derivative. We suspect this is because the
|
||
user has previously run <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--upgrade</span> <span class="pre">pip</span></code> and
|
||
therefore now has a version of <code class="docutils literal notranslate"><span class="pre">/usr/bin/pip</span></code> without the Debian
|
||
patch; standardizing this behavior in upstream package installers
|
||
would address this problem.)</p>
|
||
</li>
|
||
<li>A distro Python when used inside a virtual environment (either from
|
||
<code class="docutils literal notranslate"><span class="pre">venv</span></code> or <code class="docutils literal notranslate"><span class="pre">virtualenv</span></code>).<p>Inside a virtual environment, all packages are owned by that
|
||
environment. Even when <code class="docutils literal notranslate"><span class="pre">pip</span></code>, <code class="docutils literal notranslate"><span class="pre">setuptools</span></code>, etc. are installed
|
||
into the environment, they are and should be managed by tools
|
||
specific to that environment; they are not system-managed.</p>
|
||
</li>
|
||
<li>A distro Python when used inside a virtual environment with
|
||
<code class="docutils literal notranslate"><span class="pre">--system-site-packages</span></code>. This is like the previous case, but
|
||
worth calling out explicitly, because anything on the global
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code> is visible.<p>Currently, the answer to “Will <code class="docutils literal notranslate"><span class="pre">pip</span></code> delete externally-installed
|
||
packages” is no, because pip has a special case for running in a
|
||
virtual environment and attempting to delete packages outside it.
|
||
After this PEP, the answer remains no, but the reasoning becomes
|
||
more general: system site packages will be outside any of the
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> schemes used for package management in the
|
||
environment.</p>
|
||
</li>
|
||
<li>A distro Python when used in a single-application container image
|
||
(e.g., a Docker container). In this use case, the risk of breaking
|
||
system software is lower, since generally only a single application
|
||
runs in the container, and the impact is lower, since you can
|
||
rebuild the container and you don’t have to struggle to recover a
|
||
running machine. There are also a large number of existing
|
||
Dockerfiles with an unqualified <code class="docutils literal notranslate"><span class="pre">RUN</span> <span class="pre">pip</span> <span class="pre">install</span> <span class="pre">...</span></code> statement,
|
||
etc., and it would be good not to break those. So, builders of base
|
||
container images may want to ensure that the marker file is not
|
||
present, even if the underlying OS ships one by default.<p>There is a small behavior change: currently, <code class="docutils literal notranslate"><span class="pre">pip</span></code> run as root
|
||
will delete externally-installed packages, but after this PEP it
|
||
will not. We don’t propose a way to override this. However, since
|
||
the base image is generally minimal, there shouldn’t be much of a
|
||
use case for simply uninstalling packages (especially without using
|
||
the distro’s own tools). The common case is when pip wants to
|
||
upgrade a package, which previously would have deleted the old
|
||
version (except on Debian). After this change, the old version will
|
||
still be on disk, but pip will still <em>shadow</em> externally-installed
|
||
packages, and we believe this to be sufficient for this not to be a
|
||
breaking change in practice - a Python <code class="docutils literal notranslate"><span class="pre">import</span></code> statement will
|
||
still get you the newly-installed package.</p>
|
||
<p>If it becomes necessary to have a way to do this, we suggest that
|
||
the distro should document a way for the installer tool to access
|
||
the <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme used by the distro itself. See the
|
||
<a class="reference internal" href="#recommendations-for-distros">Recommendations for distros</a> section for more discussion.</p>
|
||
<p>It is the view of the authors of this PEP that it’s still a good
|
||
idea to use virtual environments with distro-installed Python
|
||
interpreters, even in single-application container images. Even
|
||
though they run a single <em>application</em>, that application may run
|
||
commands from the OS that are implemented in Python, and if you’ve
|
||
installed or upgraded the distro-shipped Python packages using
|
||
Python-specific tools, those commands may break.</p>
|
||
</li>
|
||
<li>Conda specifically supports the use of non-<code class="docutils literal notranslate"><span class="pre">conda</span></code> tools like pip
|
||
to install software not available in the Conda repositories. In
|
||
this context, Conda acts as the external package manager / distro
|
||
and pip as the Python-specific one.<p>In some sense, this is similar to the first case, since Conda
|
||
provides its own installation of the Python interpreter.</p>
|
||
<p>We don’t believe this PEP requires any changes to Conda, and
|
||
versions of pip that have implemented the changes in this PEP will
|
||
continue to behave as they currently do inside Conda environments.
|
||
(That said, it may be worth considering whether to use separate
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> schemes for pip-installed and Conda-installed
|
||
software, for the same reasons it’s a good idea for other distros.)</p>
|
||
</li>
|
||
<li>By a “developer-facing distro,” we mean a specific type of distro
|
||
where direct users of Python or other languages in the distro are
|
||
expected or encouraged to make changes to the distro itself if they
|
||
wish to add libraries. Common examples include private “monorepos”
|
||
at software development companies, where a single repository builds
|
||
both third-party and in-house software, and the direct users of the
|
||
distro’s Python interpreter are generally software developers
|
||
writing said in-house software. User-level package managers like
|
||
<a class="reference external" href="https://github.com/NixOS/nixpkgs">Nixpkgs</a> may also count, because they encourage users of Nix who
|
||
are Python developers to <a class="reference external" href="https://nixos.wiki/wiki/Python">package their software for Nix</a>.<p>In these cases, the distro may want to respond to an attempted
|
||
<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> with guidance encouraging use of the distro’s own
|
||
facilities for adding new packages, along with a link to
|
||
documentation.</p>
|
||
<p>If the distro supports/encourages creating a virtual environment
|
||
from the distro’s Python interpreter, there may also be custom
|
||
instructions for how to properly set up a virtual environment (as
|
||
for example Nixpkgs does).</p>
|
||
</li>
|
||
<li>When building distro Python packages for a distro Python (case 2),
|
||
it may be useful to have <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> be usable as part of the
|
||
distro’s package build process. (Consider, for instance, building a
|
||
<code class="docutils literal notranslate"><span class="pre">python-xyz</span></code> RPM by using <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">.</span></code> inside an sdist /
|
||
source tarball for <code class="docutils literal notranslate"><span class="pre">xyz</span></code>.) The distro may also want to use a more
|
||
targeted but still Python-specific installation tool such as
|
||
<a class="reference external" href="https://installer.rtfd.io/">installer</a>.<p>For this case, the build process will need to find some way to
|
||
suppress the marker file to allow <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> to work, and will
|
||
probably need to point the Python-specific tool at the distro’s
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme instead of the shipped default. See the
|
||
<a class="reference internal" href="#recommendations-for-distros">Recommendations for distros</a> section for more discussion on how
|
||
to implement this.</p>
|
||
<p>As a result of this PEP, pip will no longer be able to remove
|
||
packages already on the system. However, this behavior change is
|
||
fine because a package build process should not (and generally
|
||
cannot) include instructions to delete some other files on the
|
||
system; it can only package up its own files.</p>
|
||
</li>
|
||
<li>A distro Python used with <code class="docutils literal notranslate"><span class="pre">PYTHONHOME</span></code> to set up an alternative
|
||
Python environment (as opposed to a virtual environment), where
|
||
<code class="docutils literal notranslate"><span class="pre">PYTHONHOME</span></code> is set to some directory copied directly from the
|
||
distro Python (e.g., <code class="docutils literal notranslate"><span class="pre">cp</span> <span class="pre">-a</span> <span class="pre">/usr/lib/python3.x</span> <span class="pre">pyhome/lib</span></code>).<p>Assuming there are no modifications, then the behavior is just like
|
||
the underlying distro Python (case 2). So there are behavior
|
||
changes - you can no longer <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> by default, and if you
|
||
override it, it will no longer delete externally-installed packages
|
||
(i.e., Python packages that were copied from the OS and live in the
|
||
OS-managed <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> entry).</p>
|
||
<p>This behavior change seems to be defensible, in that if your
|
||
<code class="docutils literal notranslate"><span class="pre">PYTHONHOME</span></code> is a straight copy of the distro’s Python, it should
|
||
behave like the distro’s Python.</p>
|
||
</li>
|
||
<li>A distro Python (or any Python interpreter) used with a
|
||
<code class="docutils literal notranslate"><span class="pre">PYTHONHOME</span></code> taken from a compatible unmodified upstream Python.<p>Because the behavior changes in this PEP are keyed off of files in
|
||
the standard library (the marker file in <code class="docutils literal notranslate"><span class="pre">stdlib</span></code> and the
|
||
behavior of the <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> module), the behavior is just like
|
||
an unmodified upstream CPython (case 1).</p>
|
||
</li>
|
||
</ol>
|
||
</section>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<section id="marking-an-interpreter-as-using-an-external-package-manager">
|
||
<h3><a class="toc-backref" href="#marking-an-interpreter-as-using-an-external-package-manager" role="doc-backlink">Marking an interpreter as using an external package manager</a></h3>
|
||
<p>Before a Python-specific package installer (that is, a tool such as
|
||
pip - not an external tool such as apt) installs a package into a
|
||
certain Python context, it should make the following checks by
|
||
default:</p>
|
||
<ol class="arabic simple">
|
||
<li>Is it running outside of a virtual environment? It can determine
|
||
this by whether <code class="docutils literal notranslate"><span class="pre">sys.prefix</span> <span class="pre">==</span> <span class="pre">sys.base_prefix</span></code> (but see
|
||
<a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a>).</li>
|
||
<li>Is there an <code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> file in the directory identified
|
||
by <code class="docutils literal notranslate"><span class="pre">sysconfig.get_path("stdlib",</span>
|
||
<span class="pre">sysconfig.get_default_scheme())</span></code>?</li>
|
||
</ol>
|
||
<p>If both of these conditions are true, the installer should exit with
|
||
an error message indicating that package installation into this Python
|
||
interpreter’s directory are disabled outside of a virtual environment.</p>
|
||
<p>The installer should have a way for the user to override these rules,
|
||
such as a command-line flag <code class="docutils literal notranslate"><span class="pre">--break-system-packages</span></code>. This option
|
||
should not be enabled by default and should carry some connotation
|
||
that its use is risky.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> file is an INI-style metadata file intended
|
||
to be parsable by the standard library <a class="reference external" href="https://docs.python.org/3/library/configparser.html">configparser</a> module. If the
|
||
file can be parsed by
|
||
<code class="docutils literal notranslate"><span class="pre">configparser.ConfigParser(interpolation=None)</span></code> using the UTF-8
|
||
encoding, and it contains a section <code class="docutils literal notranslate"><span class="pre">[externally-managed]</span></code>, then the
|
||
installer should look for an error message specified in the file and
|
||
output it as part of its error. If the first element of the tuple
|
||
returned by <code class="docutils literal notranslate"><span class="pre">locale.getlocale(locale.LC_MESSAGES)</span></code>, i.e., the
|
||
language code, is not <code class="docutils literal notranslate"><span class="pre">None</span></code>, it should look for the error message
|
||
as the value of a key named <code class="docutils literal notranslate"><span class="pre">Error-</span></code> followed by the language code.
|
||
If that key does not exist, and if the language code contains
|
||
underscore or hyphen, it should look for a key named <code class="docutils literal notranslate"><span class="pre">Error-</span></code>
|
||
followed by the portion of the language code before the underscore or
|
||
hyphen. If it cannot find either of those, or if the language code is
|
||
<code class="docutils literal notranslate"><span class="pre">None</span></code>, it should look for a key simply named <code class="docutils literal notranslate"><span class="pre">Error</span></code>.</p>
|
||
<p>If the installer cannot find an error message in the file (either
|
||
because the file cannot be parsed or because no suitable error key
|
||
exists), then the installer should just use a pre-defined error
|
||
message of its own, which should suggest that the user create a
|
||
virtual environment to install packages.</p>
|
||
<p>Software distributors who have a non-Python-specific package manager
|
||
that manages libraries in the <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> of their Python package
|
||
should, in general, ship a <code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> file in their
|
||
standard library directory. For instance, Debian may ship a file in
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/lib/python3.9/EXTERNALLY-MANAGED</span></code> consisting of something like</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">externally</span><span class="o">-</span><span class="n">managed</span><span class="p">]</span>
|
||
<span class="n">Error</span><span class="o">=</span><span class="n">To</span> <span class="n">install</span> <span class="n">Python</span> <span class="n">packages</span> <span class="n">system</span><span class="o">-</span><span class="n">wide</span><span class="p">,</span> <span class="k">try</span> <span class="n">apt</span> <span class="n">install</span>
|
||
<span class="n">python3</span><span class="o">-</span><span class="n">xyz</span><span class="p">,</span> <span class="n">where</span> <span class="n">xyz</span> <span class="ow">is</span> <span class="n">the</span> <span class="n">package</span> <span class="n">you</span> <span class="n">are</span> <span class="n">trying</span> <span class="n">to</span>
|
||
<span class="n">install</span><span class="o">.</span>
|
||
|
||
<span class="n">If</span> <span class="n">you</span> <span class="n">wish</span> <span class="n">to</span> <span class="n">install</span> <span class="n">a</span> <span class="n">non</span><span class="o">-</span><span class="n">Debian</span><span class="o">-</span><span class="n">packaged</span> <span class="n">Python</span> <span class="n">package</span><span class="p">,</span>
|
||
<span class="n">create</span> <span class="n">a</span> <span class="n">virtual</span> <span class="n">environment</span> <span class="n">using</span> <span class="n">python3</span> <span class="o">-</span><span class="n">m</span> <span class="n">venv</span> <span class="n">path</span><span class="o">/</span><span class="n">to</span><span class="o">/</span><span class="n">venv</span><span class="o">.</span>
|
||
<span class="n">Then</span> <span class="n">use</span> <span class="n">path</span><span class="o">/</span><span class="n">to</span><span class="o">/</span><span class="n">venv</span><span class="o">/</span><span class="nb">bin</span><span class="o">/</span><span class="n">python</span> <span class="ow">and</span> <span class="n">path</span><span class="o">/</span><span class="n">to</span><span class="o">/</span><span class="n">venv</span><span class="o">/</span><span class="nb">bin</span><span class="o">/</span><span class="n">pip</span><span class="o">.</span> <span class="n">Make</span>
|
||
<span class="n">sure</span> <span class="n">you</span> <span class="n">have</span> <span class="n">python3</span><span class="o">-</span><span class="n">full</span> <span class="n">installed</span><span class="o">.</span>
|
||
|
||
<span class="n">If</span> <span class="n">you</span> <span class="n">wish</span> <span class="n">to</span> <span class="n">install</span> <span class="n">a</span> <span class="n">non</span><span class="o">-</span><span class="n">Debian</span> <span class="n">packaged</span> <span class="n">Python</span> <span class="n">application</span><span class="p">,</span>
|
||
<span class="n">it</span> <span class="n">may</span> <span class="n">be</span> <span class="n">easiest</span> <span class="n">to</span> <span class="n">use</span> <span class="n">pipx</span> <span class="n">install</span> <span class="n">xyz</span><span class="p">,</span> <span class="n">which</span> <span class="n">will</span> <span class="n">manage</span> <span class="n">a</span>
|
||
<span class="n">virtual</span> <span class="n">environment</span> <span class="k">for</span> <span class="n">you</span><span class="o">.</span> <span class="n">Make</span> <span class="n">sure</span> <span class="n">you</span> <span class="n">have</span> <span class="n">pipx</span> <span class="n">installed</span><span class="o">.</span>
|
||
|
||
<span class="n">See</span> <span class="o">/</span><span class="n">usr</span><span class="o">/</span><span class="n">share</span><span class="o">/</span><span class="n">doc</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">README</span><span class="o">.</span><span class="n">venv</span> <span class="k">for</span> <span class="n">more</span> <span class="n">information</span><span class="o">.</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>which provides useful and distro-relevant information
|
||
to a user trying to install a package. Optionally,
|
||
translations can be provided in the same file:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Error-de_DE=Wenn ist das Nunstück git und Slotermeyer?
|
||
|
||
Ja! Beiherhund das Oder die Virtualenvironment gersput!
|
||
</pre></div>
|
||
</div>
|
||
<p>In certain contexts, such as single-application container images that
|
||
aren’t updated after creation, a distributor may choose not to ship an
|
||
<code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> file, so that users can install whatever they
|
||
like (as they can today) without having to manually override this
|
||
rule.</p>
|
||
</section>
|
||
<section id="writing-to-only-the-target-sysconfig-scheme">
|
||
<h3><a class="toc-backref" href="#writing-to-only-the-target-sysconfig-scheme" role="doc-backlink">Writing to only the target <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme</a></h3>
|
||
<p>Usually, a Python package installer installs to directories in a
|
||
scheme returned by the <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> standard library package.
|
||
Ordinarily, this is the scheme returned by
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig.get_default_scheme()</span></code>, but based on configuration (e.g.
|
||
<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code>), it may use a different scheme.</p>
|
||
<p>Whenever the installer is installing to a <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme, this
|
||
PEP specifies that the installer should never modify or delete files
|
||
outside of that scheme. For instance, if it’s upgrading a package, and
|
||
the package is already installed in a directory outside that scheme
|
||
(perhaps in a directory from another scheme), it should leave the
|
||
existing files alone.</p>
|
||
<p>If the installer does end up shadowing an existing installation during
|
||
an upgrade, we recommend that it produces a warning at the end of its
|
||
run.</p>
|
||
<p>If the installer is installing to a location outside of a
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme (e.g., <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--target</span></code>), then this
|
||
subsection does not apply.</p>
|
||
</section>
|
||
</section>
|
||
<section id="recommendations-for-distros">
|
||
<h2><a class="toc-backref" href="#recommendations-for-distros" role="doc-backlink">Recommendations for distros</a></h2>
|
||
<p>This section is non-normative. It provides best practices we believe
|
||
distros should follow unless they have a specific reason otherwise.</p>
|
||
<section id="mark-the-installation-as-externally-managed">
|
||
<h3><a class="toc-backref" href="#mark-the-installation-as-externally-managed" role="doc-backlink">Mark the installation as externally managed</a></h3>
|
||
<p>Distros should create an <code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> file in their
|
||
<code class="docutils literal notranslate"><span class="pre">stdlib</span></code> directory.</p>
|
||
</section>
|
||
<section id="guide-users-towards-virtual-environments">
|
||
<h3><a class="toc-backref" href="#guide-users-towards-virtual-environments" role="doc-backlink">Guide users towards virtual environments</a></h3>
|
||
<p>The file should contain a useful and distro-relevant error message
|
||
indicating both how to install system-wide packages via the distro’s
|
||
package manager and how to set up a virtual environment. If your
|
||
distro is often used by users in a state where the <code class="docutils literal notranslate"><span class="pre">python3</span></code> command
|
||
is available (and especially where <code class="docutils literal notranslate"><span class="pre">pip</span></code> or <code class="docutils literal notranslate"><span class="pre">get-pip</span></code> is
|
||
available) but <code class="docutils literal notranslate"><span class="pre">python3</span> <span class="pre">-m</span> <span class="pre">venv</span></code> does not work, the message should
|
||
indicate clearly how to make <code class="docutils literal notranslate"><span class="pre">python3</span> <span class="pre">-m</span> <span class="pre">venv</span></code> work properly.</p>
|
||
<p>Consider packaging <a class="reference external" href="https://github.com/pypa/pipx">pipx</a>, a tool for installing Python-language
|
||
applications, and suggesting it in the error. pipx automatically
|
||
creates a virtual environment for that application alone, which is a
|
||
much better default for end users who want to install some
|
||
Python-language software (which isn’t available in the distro) but are
|
||
not themselves Python users. Packaging pipx in the distro avoids the
|
||
irony of instructing users to <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span>
|
||
<span class="pre">--break-system-packages</span> <span class="pre">pipx</span></code> to <em>avoid</em> breaking system packages.
|
||
Consider arranging things so your distro’s package / environment for
|
||
Python for end users (e.g., <code class="docutils literal notranslate"><span class="pre">python3</span></code> on Fedora or <code class="docutils literal notranslate"><span class="pre">python3-full</span></code>
|
||
on Debian) depends on pipx.</p>
|
||
</section>
|
||
<section id="keep-the-marker-file-in-container-images">
|
||
<h3><a class="toc-backref" href="#keep-the-marker-file-in-container-images" role="doc-backlink">Keep the marker file in container images</a></h3>
|
||
<p>Distros that produce official images for single-application containers
|
||
(e.g., Docker container images) should keep the
|
||
<code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> file, preferably in a way that makes it not
|
||
go away if a user of that image installs package updates inside
|
||
their image (think <code class="docutils literal notranslate"><span class="pre">RUN</span> <span class="pre">apt-get</span> <span class="pre">dist-upgrade</span></code>).</p>
|
||
</section>
|
||
<section id="create-separate-distro-and-local-directories">
|
||
<h3><a class="toc-backref" href="#create-separate-distro-and-local-directories" role="doc-backlink">Create separate distro and local directories</a></h3>
|
||
<p>Distros should place two separate paths on the system interpreter’s
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, one for distro-installed packages and one for packages
|
||
installed by the local system administrator, and configure
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig.get_default_scheme()</span></code> to point at the latter path. This
|
||
ensures that tools like pip will not modify distro-installed packages.
|
||
The path for the local system administrator should come before the
|
||
distro path on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> so that local installs take preference
|
||
over distro packages.</p>
|
||
<p>For example, Fedora and Debian (and their derivatives) both implement
|
||
this split by using <code class="docutils literal notranslate"><span class="pre">/usr/local</span></code> for locally-installed packages and
|
||
<code class="docutils literal notranslate"><span class="pre">/usr</span></code> for distro-installed packages. Fedora uses
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/local/lib/python3.x/site-packages</span></code> vs.
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/lib/python3.x/site-packages</span></code>. (Debian uses
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/local/lib/python3/dist-packages</span></code> vs.
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/lib/python3/dist-packages</span></code> as an additional layer of
|
||
separation from a locally-compiled Python interpreter: if you build
|
||
and install upstream CPython in <code class="docutils literal notranslate"><span class="pre">/usr/local/bin</span></code>, it will look at
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/local/lib/python3/site-packages</span></code>, and Debian wishes to make
|
||
sure that packages installed via the locally-built interpreter don’t
|
||
show up on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> for the distro interpreter.)</p>
|
||
<p>Note that the <code class="docutils literal notranslate"><span class="pre">/usr/local</span></code> vs. <code class="docutils literal notranslate"><span class="pre">/usr</span></code> split is analogous to how
|
||
the <code class="docutils literal notranslate"><span class="pre">PATH</span></code> environment variable typically includes
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/local/bin:/usr/bin</span></code> and non-distro software installs to
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/local</span></code> by default. This split is <a class="reference external" href="https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch04s09.html">recommended by the
|
||
Filesystem Hierarchy Standard</a>.</p>
|
||
<p>There are two ways you could do this. One is, if you are building and
|
||
packaging Python libraries directly (e.g., your packaging helpers
|
||
unpack a <a class="pep reference internal" href="../pep-0517/" title="PEP 517 – A build-system independent format for source trees">PEP 517</a>-built wheel or call <code class="docutils literal notranslate"><span class="pre">setup.py</span> <span class="pre">install</span></code>), arrange
|
||
for those tools to use a directory that is not in a <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code>
|
||
scheme but is still on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>.</p>
|
||
<p>The other is to arrange for the default <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme to change
|
||
when running inside a package build versus when running on an
|
||
installed system. The <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> customization hooks from
|
||
<a class="reference external" href="https://bugs.python.org/issue43976">bpo-43976</a> should make this easy (once accepted and implemented):
|
||
make your packaging tool set an
|
||
environment variable or some other detectable configuration, and
|
||
define a <code class="docutils literal notranslate"><span class="pre">get_preferred_schemes</span></code> function to return a different
|
||
scheme when called from inside a package build. Then you can use <code class="docutils literal notranslate"><span class="pre">pip</span>
|
||
<span class="pre">install</span></code> as part of your distro packaging.</p>
|
||
<p>We propose adding a <code class="docutils literal notranslate"><span class="pre">--scheme=...</span></code> option to instruct pip to run
|
||
against a specific scheme. (See <a class="reference internal" href="#implementation-notes">Implementation Notes</a> below for how
|
||
pip currently determines schemes.) Once that’s available, for local
|
||
testing and possibly for actual packaging, you would be able to run
|
||
something like <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--scheme=posix_distro</span></code> to explicitly
|
||
install a package into your distro’s location (bypassing
|
||
<code class="docutils literal notranslate"><span class="pre">get_preferred_schemes</span></code>). One could also, if absolutely needed, use
|
||
<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">uninstall</span> <span class="pre">--scheme=posix_distro</span></code> to use pip to remove packages
|
||
from the system-managed directory, which addresses the (hopefully
|
||
theoretical) regression in use case 5 in <a class="reference internal" href="#rationale">Rationale</a>.</p>
|
||
<p>To install packages with pip, you would also need to either suppress
|
||
the <code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> marker file to allow pip to run or to
|
||
override it on the command line. You may want to use the same means
|
||
for suppressing the marker file in build chroots as you do in
|
||
container images.</p>
|
||
<p>The advantage of setting these up to be automatic (suppressing the
|
||
marker file in your build environment and having
|
||
<code class="docutils literal notranslate"><span class="pre">get_preferred_schemes</span></code> automatically return your distro’s scheme)
|
||
is that an unadorned <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> will work inside a package build,
|
||
which generally means that an unmodified upstream build script that
|
||
happens to internally call <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> will do the right thing.
|
||
You can, of course, just ensure that your packaging process always
|
||
calls <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--scheme=posix_distro</span> <span class="pre">--break-system-packages</span></code>,
|
||
which would work too.</p>
|
||
<p>The best approach here depends a lot on your distro’s conventions and
|
||
mechanisms for packaging.</p>
|
||
<p>Similarly, the <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> paths that are not for importable Python
|
||
code - that is, <code class="docutils literal notranslate"><span class="pre">include</span></code>, <code class="docutils literal notranslate"><span class="pre">platinclude</span></code>, <code class="docutils literal notranslate"><span class="pre">scripts</span></code>, and
|
||
<code class="docutils literal notranslate"><span class="pre">data</span></code> - should also have two variants, one for use by
|
||
distro-packaged software and one for use for locally-installed
|
||
software, and the distro should be set up such that both are usable.
|
||
For instance, a typical FHS-compliant distro will use
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/local/include</span></code> for the default scheme’s <code class="docutils literal notranslate"><span class="pre">include</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/include</span></code> for distro-packaged headers and place both on the
|
||
compiler’s search path, and it will use <code class="docutils literal notranslate"><span class="pre">/usr/local/bin</span></code> for the
|
||
default scheme’s <code class="docutils literal notranslate"><span class="pre">scripts</span></code> and <code class="docutils literal notranslate"><span class="pre">/usr/bin</span></code> for distro-packaged
|
||
entry points and place both on <code class="docutils literal notranslate"><span class="pre">$PATH</span></code>.</p>
|
||
</section>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards Compatibility</a></h2>
|
||
<p>All of these mechanisms are proposed for new distro releases and new
|
||
versions of tools like pip only.</p>
|
||
<p>In particular, we strongly recommend that distros with a concept of
|
||
major versions only add the marker file or change <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code>
|
||
schemes in a new major version; otherwise there is a risk that, on an
|
||
existing system, software installed via a Python-specific package
|
||
manager now becomes unmanageable (without an override option). For a
|
||
rolling-release distro, if possible, only add the marker file or
|
||
change <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> schemes in a new Python minor version.</p>
|
||
<p>One particular backwards-compatibility difficulty for package
|
||
installation tools is likely to be managing environments created by
|
||
old versions of <code class="docutils literal notranslate"><span class="pre">virtualenv</span></code> which have the latest version of the
|
||
tool installed. A “virtual environment” now has a fairly precise
|
||
definition: it uses the <code class="docutils literal notranslate"><span class="pre">pyvenv.cfg</span></code> mechanism, which causes
|
||
<code class="docutils literal notranslate"><span class="pre">sys.base_prefix</span> <span class="pre">!=</span> <span class="pre">sys.prefix</span></code>. It is possible, however, that a
|
||
user may have an old virtual environment created by an older version
|
||
of <code class="docutils literal notranslate"><span class="pre">virtualenv</span></code>; as of this writing, pip supports Python 3.6
|
||
onwards, which is in turn supported by <code class="docutils literal notranslate"><span class="pre">virtualenv</span></code> 15.1.0 onwards,
|
||
so this scenario is possible. In older versions of <code class="docutils literal notranslate"><span class="pre">virtualenv</span></code>, the
|
||
mechanism is instead to set a new attribute, <code class="docutils literal notranslate"><span class="pre">sys.real_prefix</span></code>, and
|
||
it does not use the standard library support for virtual environments,
|
||
so <code class="docutils literal notranslate"><span class="pre">sys.base_prefix</span></code> is the same as <code class="docutils literal notranslate"><span class="pre">sys.prefix</span></code>. So the logic for
|
||
robustly detecting a virtual environment is something like:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">is_virtual_environment</span><span class="p">():</span>
|
||
<span class="k">return</span> <span class="n">sys</span><span class="o">.</span><span class="n">base_prefix</span> <span class="o">!=</span> <span class="n">sys</span><span class="o">.</span><span class="n">prefix</span> <span class="ow">or</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">sys</span><span class="p">,</span> <span class="s2">"real_prefix"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="security-implications">
|
||
<h2><a class="toc-backref" href="#security-implications" role="doc-backlink">Security Implications</a></h2>
|
||
<p>The purpose of this feature is not to implement a security boundary;
|
||
it is to discourage well-intended changes from unexpectedly breaking a
|
||
user’s environment. That is to say, the reason this PEP restricts
|
||
<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> outside a virtual environment is not that it’s a
|
||
security risk to be able to do so; it’s that “There should be one–
|
||
and preferably only one –obvious way to do it,” and that way should
|
||
be using a virtual environment. <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> outside a virtual
|
||
environment is rather too obvious for what is almost always the wrong
|
||
way to do it.</p>
|
||
<p>If there is a case where a user should not be able to <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span>
|
||
<span class="pre">install</span></code> or <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code> and add files to <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> <em>for
|
||
security reasons</em>, that needs to be implemented either via access
|
||
control rules on what files the user can write to or an explicitly
|
||
secured <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> for the program in question. Neither of the
|
||
mechanisms in this PEP should be interpreted as a way to address such
|
||
a scenario.</p>
|
||
<p>For those reasons, an attempted install with a marker file present is
|
||
not a security incident, and there is no need to raise an auditing
|
||
event for it. If the calling user legitimately has access to <code class="docutils literal notranslate"><span class="pre">sudo</span>
|
||
<span class="pre">pip</span> <span class="pre">install</span></code> or <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code>, they can accomplish the same
|
||
installation entirely outside of Python; if they do not legitimately
|
||
have such access, that’s a problem outside the scope of this PEP.</p>
|
||
<p>The marker file itself is located in the standard library directory,
|
||
which is a trusted location (i.e., anyone who can write to the marker
|
||
file used by a particular installer could, presumably, run arbitrary
|
||
code inside the installer). Therefore, there is generally no need to
|
||
filter out terminal escape sequences or other potentially-malicious
|
||
content in the error message.</p>
|
||
</section>
|
||
<section id="alternatives">
|
||
<h2><a class="toc-backref" href="#alternatives" role="doc-backlink">Alternatives</a></h2>
|
||
<p>There are a number of similar proposals we considered that this PEP
|
||
rejects or defers, largely to preserve the behavior in the
|
||
case-by-case analysis in <a class="reference internal" href="#rationale">Rationale</a>.</p>
|
||
<section id="marker-file">
|
||
<h3><a class="toc-backref" href="#marker-file" role="doc-backlink">Marker file</a></h3>
|
||
<p>Should the marker file be in <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, marking a particular
|
||
directory as not to be written to by a Python-specific package
|
||
manager? This would help with the second problem addressed by this PEP
|
||
(not overwriting deleting distro-owned files) but not the first
|
||
(incompatible installs). A directory-specific marker in
|
||
<code class="docutils literal notranslate"><span class="pre">/usr/lib/python3.x/site-packages</span></code> would not discourage
|
||
installations into either <code class="docutils literal notranslate"><span class="pre">/usr/local/lib/python3.x/site-packages</span></code>
|
||
or <code class="docutils literal notranslate"><span class="pre">~/.local/lib/python3.x/site-packages</span></code>, both of which are on
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code> for <code class="docutils literal notranslate"><span class="pre">/usr/bin/python3</span></code>. In other words, the marker file
|
||
should not be interpreted as marking a single <em>directory</em> as
|
||
externally managed (even though it happens to be in a directory on
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>); it marks the entire <em>Python installation</em> as externally
|
||
managed.</p>
|
||
<p>Another variant of the above: should the marker file be in
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, where if it can be found in any directory in
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, it marks the installation as externally managed? An
|
||
apparent advantage of this approach is that it automatically disables
|
||
itself in virtual environments. Unfortunately, This has the wrong
|
||
behavior with a <code class="docutils literal notranslate"><span class="pre">--system-site-packages</span></code> virtual environment, where
|
||
the system-wide <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> is visible but package installations are
|
||
allowed. (It could work if the rule of exempting virtual environments
|
||
is preserved, but that seems to have no advantage over the current
|
||
scheme.)</p>
|
||
<p>Should the marker just be a new attribute of a <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme?
|
||
There is some conceptual cleanliness to this, except that it’s hard to
|
||
override. We want to make it easy for container images, package build
|
||
environments, etc. to suppress the marker file. A file that you can
|
||
remove is easy; code in <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> is much harder to modify.</p>
|
||
<p>Should the file be in <code class="docutils literal notranslate"><span class="pre">/etc</span></code>? No, because again, it refers to a
|
||
specific Python installation. A user who installs their own Python may
|
||
well want to install packages within the global context of that
|
||
interpreter.</p>
|
||
<p>Should the configuration setting be in <code class="docutils literal notranslate"><span class="pre">pip.conf</span></code> or
|
||
<code class="docutils literal notranslate"><span class="pre">distutils.cfg</span></code>? Apart from the above objections about marking an
|
||
installation, this mechanism isn’t specific to either of those tools.
|
||
(It seems reasonable for pip to <em>also</em> implement a configuration flag
|
||
for users to prevent themselves from performing accidental
|
||
non-virtual-environment installs in any Python installation, but that
|
||
is outside the scope of this PEP.)</p>
|
||
<p>Should the file be TOML? TOML is gaining popularity for packaging (see
|
||
e.g. <a class="pep reference internal" href="../pep-0517/" title="PEP 517 – A build-system independent format for source trees">PEP 517</a>) but does not yet have an implementation in the standard
|
||
library. Strictly speaking, this isn’t a blocker - distros need only
|
||
write the file, not read it, so they don’t need a TOML library (the
|
||
file will probably be written by hand, regardless of format), and
|
||
packaging tools likely have a TOML reader already. However, the INI
|
||
format is currently used for various other forms of packaging metadata
|
||
(e.g., <code class="docutils literal notranslate"><span class="pre">pydistutils.cfg</span></code> and <code class="docutils literal notranslate"><span class="pre">setup.cfg</span></code>), meets our needs, and is
|
||
parsable by the standard library, and the pip maintainers expressed a
|
||
preference to avoid using TOML for this yet.</p>
|
||
<p>Should the file be <code class="docutils literal notranslate"><span class="pre">email.message</span></code>-style? While this format is also
|
||
used for packaging metadata (e.g. sdist and wheel metadata) and is
|
||
also parsable by the standard library, it doesn’t handle multi-line
|
||
entries quite as clearly, and that is our primary use case.</p>
|
||
<p>Should the marker file be executable Python code that evaluates
|
||
whether installation should be allowed or not? Apart from the concerns
|
||
above about having the file in <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, we have a concern that
|
||
making it executable is committing to too powerful of an API and risks
|
||
making behavior harder to understand. (Note that the
|
||
<code class="docutils literal notranslate"><span class="pre">get_default_scheme</span></code> hook of <a class="reference external" href="https://bugs.python.org/issue43976">bpo-43976</a> is in fact executable, but
|
||
that code needs to be supplied when the interpreter builds; it isn’t
|
||
intended to be supplied post-build.)</p>
|
||
<p>When overriding the marker, should a Python-specific package manager
|
||
be disallowed from shadowing a package installed by the external
|
||
package manager (i.e., installing modules of the same name)? This
|
||
would minimize the risk of breaking system software, but it’s not
|
||
clear it’s worth the additional user experience complexity. There are
|
||
legitimate use cases for shadowing system packages, and an additional
|
||
command-line option to permit it would be more confusing. Meanwhile,
|
||
not passing that option wouldn’t eliminate the risk of breaking system
|
||
software, which may be relying on a <code class="docutils literal notranslate"><span class="pre">try:</span> <span class="pre">import</span> <span class="pre">xyz</span></code> failing,
|
||
finding a limited set of entry points, etc. Communicating this
|
||
distinction seems difficult. We think it’s a good idea for
|
||
Python-specific package managers to print a warning if they shadow a
|
||
package, but we think it’s not worth disabling it by default.</p>
|
||
<p>Why not use the <code class="docutils literal notranslate"><span class="pre">INSTALLER</span></code> file from <a class="pep reference internal" href="../pep-0376/" title="PEP 376 – Database of Installed Python Distributions">PEP 376</a> to determine who
|
||
installed a package and whether it can be removed? First, it’s
|
||
specific to a particular package (it’s in the package’s <code class="docutils literal notranslate"><span class="pre">dist-info</span></code>
|
||
directory), so like some of the alternatives above, it doesn’t provide
|
||
information on an entire environment and whether package installations
|
||
are permissible. <a class="pep reference internal" href="../pep-0627/" title="PEP 627 – Recording installed projects">PEP 627</a> also updates <a class="pep reference internal" href="../pep-0376/" title="PEP 376 – Database of Installed Python Distributions">PEP 376</a> to prevent programmatic
|
||
use of <code class="docutils literal notranslate"><span class="pre">INSTALLER</span></code>, specifying that the file is “to be used for
|
||
informational purposes only. […] Our goal is supporting
|
||
interoperating tools, and basing any action on which tool happened to
|
||
install a package runs counter to that goal.” Finally, as <a class="pep reference internal" href="../pep-0627/" title="PEP 627 – Recording installed projects">PEP 627</a>
|
||
envisions, there are legitimate use cases for one tool knowing how to
|
||
handle packages installed by another tool; for instance, <code class="docutils literal notranslate"><span class="pre">conda</span></code> can
|
||
safely remove a package installed by <code class="docutils literal notranslate"><span class="pre">pip</span></code> into a Conda environment.</p>
|
||
<p>Why does the specification give no means for disabling package
|
||
installations inside a virtual environment? We can’t see a
|
||
particularly strong use case for it (at least not one related to the
|
||
purposes of this PEP). If you need it, it’s simple enough to <code class="docutils literal notranslate"><span class="pre">pip</span>
|
||
<span class="pre">uninstall</span> <span class="pre">pip</span></code> inside that environment, which should discourage at
|
||
least unintentional changes to the environment (and this specification
|
||
makes no provision to disable <em>intentional</em> changes, since after all
|
||
the marker file can be easily removed).</p>
|
||
</section>
|
||
<section id="system-python">
|
||
<h3><a class="toc-backref" href="#system-python" role="doc-backlink">System Python</a></h3>
|
||
<p>Shouldn’t distro software just run with the distro <code class="docutils literal notranslate"><span class="pre">site-packages</span></code>
|
||
directory alone on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> and ignore the local system
|
||
administrator’s <code class="docutils literal notranslate"><span class="pre">site-packages</span></code> as well as the user-specific one?
|
||
This is a worthwhile idea, and various versions of it have been
|
||
circulating for a while under the name of “system Python” or “platform
|
||
Python” (with a separate “user Python” for end users writing Python or
|
||
installing Python software separate from the system). However, it’s
|
||
much more involved of a change. First, it would be a
|
||
backwards-incompatible change. As mentioned in the <a class="reference internal" href="#motivation">Motivation</a>
|
||
section, there are valid use cases for running distro-installed Python
|
||
applications like Sphinx or Ansible with locally-installed Python
|
||
libraries available on their <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>. A wholesale switch to
|
||
ignoring local packages would break these use cases, and a distro
|
||
would have to make a case-by-case analysis of whether an application
|
||
ought to see locally-installed libraries or not.</p>
|
||
<p>Furthermore, <a class="reference external" href="https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/SEFUWW4XZBTVOAQ36XOJQ72PIICMFOSN/">Fedora attempted this change and reverted it</a>, finding,
|
||
ironically, that their implementation of the change <a class="reference external" href="https://bugzilla.redhat.com/show_bug.cgi?id=1483342">broke their
|
||
package manager</a>. Given that experience, there are clearly details to
|
||
be worked out before distros can reliably implement that approach, and
|
||
a PEP recommending it would be premature.</p>
|
||
<p>This PEP is intended to be a complete and self-contained change that
|
||
is independent of a distributor’s decision for or against “system
|
||
Python” or similar proposals. It is not incompatible with a distro
|
||
implementing “system Python” in the future, and even though both
|
||
proposals address the same class of problems, there are still
|
||
arguments in favor of implementing something like “system Python” even
|
||
after implementing this PEP. At the same time, though, this PEP
|
||
specifically tries to make a more targeted and minimal change, such
|
||
that it can be implemented by distributors who don’t expect to adopt
|
||
“system Python” (or don’t expect to implement it immediately). The
|
||
changes in this PEP stand on their own merits and are not an
|
||
intermediate step for some future proposal. This PEP reduces (but does
|
||
not eliminate) the risk of breaking system software while minimizing
|
||
(but not completely avoiding) breaking changes, which should therefore
|
||
be much easier to implement than the full “system Python” idea, which
|
||
comes with the downsides mentioned above.</p>
|
||
<p>We expect that the guidance in this PEP - that users should use
|
||
virtual environments whenever possible and that distros should have
|
||
separate <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> directories for distro-managed and
|
||
locally-managed modules - should make further experiments easier in
|
||
the future. These may include distributing wholly separate “system”
|
||
and “user” Python interpreters, running system software out of a
|
||
distro-owned virtual environment or <code class="docutils literal notranslate"><span class="pre">PYTHONHOME</span></code> (but shipping a
|
||
single interpreter), or modifying the entry points for certain
|
||
software (such as the distro’s package manager) to use a <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>
|
||
that only sees distro-managed directories. Those ideas themselves,
|
||
however, remain outside the scope of this PEP.</p>
|
||
</section>
|
||
</section>
|
||
<section id="implementation-notes">
|
||
<h2><a class="toc-backref" href="#implementation-notes" role="doc-backlink">Implementation Notes</a></h2>
|
||
<p>This section is non-normative and contains notes relevant to both the
|
||
specification and potential implementations.</p>
|
||
<p>Currently, pip does not directly expose a way to choose a target
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme, but it has three ways of looking up schemes when
|
||
installing:</p>
|
||
<dl class="simple">
|
||
<dt><code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code></dt><dd>Calls <code class="docutils literal notranslate"><span class="pre">sysconfig.get_default_scheme()</span></code>, which is usually (in
|
||
upstream CPython and most current distros) the same as
|
||
<code class="docutils literal notranslate"><span class="pre">get_preferred_scheme('prefix')</span></code>.</dd>
|
||
<dt><code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--prefix=/some/path</span></code></dt><dd>Calls <code class="docutils literal notranslate"><span class="pre">sysconfig.get_preferred_scheme('prefix')</span></code>.</dd>
|
||
<dt><code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code></dt><dd>Calls <code class="docutils literal notranslate"><span class="pre">sysconfig.get_preferred_scheme('user')</span></code>.</dd>
|
||
</dl>
|
||
<p>Finally, <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--target=/some/path</span></code> writes directly to
|
||
<code class="docutils literal notranslate"><span class="pre">/some/path</span></code> without looking up any schemes.</p>
|
||
<p>Debian currently carries a <a class="reference external" href="https://sources.debian.org/src/python3.7/3.7.3-2+deb10u3/debian/patches/distutils-install-layout.diff/">patch to change the default install
|
||
location inside a virtual environment</a>, using a few heuristics
|
||
(including checking for the <code class="docutils literal notranslate"><span class="pre">VIRTUAL_ENV</span></code> environment variable),
|
||
largely so that the directory used in a virtual environment remains
|
||
<code class="docutils literal notranslate"><span class="pre">site-packages</span></code> and not <code class="docutils literal notranslate"><span class="pre">dist-packages</span></code>. This does not
|
||
particularly affect this proposal, because the implementation of that
|
||
patch does not actually change the default <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme, and
|
||
notably does not change the result of
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig.get_path("stdlib")</span></code>.</p>
|
||
<p>Fedora currently carries a <a class="reference external" href="https://src.fedoraproject.org/rpms/python3.9/blob/f34/f/00251-change-user-install-location.patch">patch to change the default install
|
||
location when not running inside rpmbuild</a>, which they use to
|
||
implement the two-system-wide-directories approach. This is
|
||
conceptually the sort of hook envisioned by <a class="reference external" href="https://bugs.python.org/issue43976">bpo-43976</a>, except
|
||
implemented as a code patch to <code class="docutils literal notranslate"><span class="pre">distutils</span></code> instead of as a changed
|
||
<code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme.</p>
|
||
<p>The implementation of <code class="docutils literal notranslate"><span class="pre">is_virtual_environment</span></code> above, as well as the
|
||
logic to load the <code class="docutils literal notranslate"><span class="pre">EXTERNALLY-MANAGED</span></code> file and find the error
|
||
message from it, may as well get added to the standard library
|
||
(<code class="docutils literal notranslate"><span class="pre">sys</span></code> and <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code>, respectively), to centralize their
|
||
implementations, but they don’t need to be added yet.</p>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<p>For additional background on these problems and previous attempts to
|
||
solve them, see <a class="reference external" href="https://bugs.debian.org/771794">Debian bug 771794</a> “pip silently removes/updates
|
||
system provided python packages” from 2014, Fedora’s 2018 article
|
||
<a class="reference external" href="https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe">Making sudo pip safe</a> about pointing <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span></code> at /usr/local
|
||
(which acknowledges that the changes still do not make <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span></code>
|
||
completely safe), pip issues <a class="reference external" href="https://github.com/pypa/pip/issues/5605">5605</a> (“Disable upgrades to existing
|
||
python modules which were not installed via pip”) and <a class="reference external" href="https://github.com/pypa/pip/issues/5722">5722</a> (“pip
|
||
should respect /usr/local”) from 2018, and the post-PyCon US 2019
|
||
discussion thread <a class="reference external" href="https://discuss.python.org/t/playing-nice-with-external-package-managers/1968">Playing nice with external package managers</a>.</p>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="pip" role="doc-footnote">
|
||
<dt class="label" id="pip">[<a href="#id1">1</a>]</dt>
|
||
<dd><a class="reference external" href="https://pip.pypa.io/en/stable/">https://pip.pypa.io/en/stable/</a></aside>
|
||
<aside class="footnote brackets" id="easy-install" role="doc-footnote">
|
||
<dt class="label" id="easy-install">[<a href="#id2">2</a>]</dt>
|
||
<dd><a class="reference external" href="https://setuptools.readthedocs.io/en/latest/deprecated/easy_install.html">https://setuptools.readthedocs.io/en/latest/deprecated/easy_install.html</a>
|
||
(Note that the <code class="docutils literal notranslate"><span class="pre">easy_install</span></code> command was removed in
|
||
setuptools version 52, released 23 January 2021.)</aside>
|
||
<aside class="footnote brackets" id="conda" role="doc-footnote">
|
||
<dt class="label" id="conda">[<a href="#id3">3</a>]</dt>
|
||
<dd><a class="reference external" href="https://conda.io">https://conda.io</a></aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document is placed in the public domain or under the
|
||
CC0-1.0-Universal license, whichever is more permissive.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0668.rst">https://github.com/python/peps/blob/main/peps/pep-0668.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0668.rst">2023-12-07 19:59:56 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#terminology">Terminology</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a><ul>
|
||
<li><a class="reference internal" href="#use-cases">Use cases</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#marking-an-interpreter-as-using-an-external-package-manager">Marking an interpreter as using an external package manager</a></li>
|
||
<li><a class="reference internal" href="#writing-to-only-the-target-sysconfig-scheme">Writing to only the target <code class="docutils literal notranslate"><span class="pre">sysconfig</span></code> scheme</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#recommendations-for-distros">Recommendations for distros</a><ul>
|
||
<li><a class="reference internal" href="#mark-the-installation-as-externally-managed">Mark the installation as externally managed</a></li>
|
||
<li><a class="reference internal" href="#guide-users-towards-virtual-environments">Guide users towards virtual environments</a></li>
|
||
<li><a class="reference internal" href="#keep-the-marker-file-in-container-images">Keep the marker file in container images</a></li>
|
||
<li><a class="reference internal" href="#create-separate-distro-and-local-directories">Create separate distro and local directories</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#security-implications">Security Implications</a></li>
|
||
<li><a class="reference internal" href="#alternatives">Alternatives</a><ul>
|
||
<li><a class="reference internal" href="#marker-file">Marker file</a></li>
|
||
<li><a class="reference internal" href="#system-python">System Python</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#implementation-notes">Implementation Notes</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0668.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |