build_env.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. """Build Environment used for isolation during sdist building
  2. """
  3. import contextlib
  4. import logging
  5. import os
  6. import pathlib
  7. import sys
  8. import textwrap
  9. import zipfile
  10. from collections import OrderedDict
  11. from sysconfig import get_paths
  12. from types import TracebackType
  13. from typing import TYPE_CHECKING, Iterable, Iterator, List, Optional, Set, Tuple, Type
  14. from pip._vendor.certifi import where
  15. from pip._vendor.packaging.requirements import Requirement
  16. from pip._vendor.packaging.version import Version
  17. from pip import __file__ as pip_location
  18. from pip._internal.cli.spinners import open_spinner
  19. from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
  20. from pip._internal.metadata import get_environment
  21. from pip._internal.utils.subprocess import call_subprocess
  22. from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
  23. if TYPE_CHECKING:
  24. from pip._internal.index.package_finder import PackageFinder
  25. logger = logging.getLogger(__name__)
  26. class _Prefix:
  27. def __init__(self, path: str) -> None:
  28. self.path = path
  29. self.setup = False
  30. self.bin_dir = get_paths(
  31. "nt" if os.name == "nt" else "posix_prefix",
  32. vars={"base": path, "platbase": path},
  33. )["scripts"]
  34. self.lib_dirs = get_prefixed_libs(path)
  35. @contextlib.contextmanager
  36. def _create_standalone_pip() -> Iterator[str]:
  37. """Create a "standalone pip" zip file.
  38. The zip file's content is identical to the currently-running pip.
  39. It will be used to install requirements into the build environment.
  40. """
  41. source = pathlib.Path(pip_location).resolve().parent
  42. # Return the current instance if `source` is not a directory. We can't build
  43. # a zip from this, and it likely means the instance is already standalone.
  44. if not source.is_dir():
  45. yield str(source)
  46. return
  47. with TempDirectory(kind="standalone-pip") as tmp_dir:
  48. pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip")
  49. kwargs = {}
  50. if sys.version_info >= (3, 8):
  51. kwargs["strict_timestamps"] = False
  52. with zipfile.ZipFile(pip_zip, "w", **kwargs) as zf:
  53. for child in source.rglob("*"):
  54. zf.write(child, child.relative_to(source.parent).as_posix())
  55. yield os.path.join(pip_zip, "pip")
  56. class BuildEnvironment:
  57. """Creates and manages an isolated environment to install build deps"""
  58. def __init__(self) -> None:
  59. temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
  60. self._prefixes = OrderedDict(
  61. (name, _Prefix(os.path.join(temp_dir.path, name)))
  62. for name in ("normal", "overlay")
  63. )
  64. self._bin_dirs: List[str] = []
  65. self._lib_dirs: List[str] = []
  66. for prefix in reversed(list(self._prefixes.values())):
  67. self._bin_dirs.append(prefix.bin_dir)
  68. self._lib_dirs.extend(prefix.lib_dirs)
  69. # Customize site to:
  70. # - ensure .pth files are honored
  71. # - prevent access to system site packages
  72. system_sites = {
  73. os.path.normcase(site) for site in (get_purelib(), get_platlib())
  74. }
  75. self._site_dir = os.path.join(temp_dir.path, "site")
  76. if not os.path.exists(self._site_dir):
  77. os.mkdir(self._site_dir)
  78. with open(
  79. os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
  80. ) as fp:
  81. fp.write(
  82. textwrap.dedent(
  83. """
  84. import os, site, sys
  85. # First, drop system-sites related paths.
  86. original_sys_path = sys.path[:]
  87. known_paths = set()
  88. for path in {system_sites!r}:
  89. site.addsitedir(path, known_paths=known_paths)
  90. system_paths = set(
  91. os.path.normcase(path)
  92. for path in sys.path[len(original_sys_path):]
  93. )
  94. original_sys_path = [
  95. path for path in original_sys_path
  96. if os.path.normcase(path) not in system_paths
  97. ]
  98. sys.path = original_sys_path
  99. # Second, add lib directories.
  100. # ensuring .pth file are processed.
  101. for path in {lib_dirs!r}:
  102. assert not path in sys.path
  103. site.addsitedir(path)
  104. """
  105. ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
  106. )
  107. def __enter__(self) -> None:
  108. self._save_env = {
  109. name: os.environ.get(name, None)
  110. for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
  111. }
  112. path = self._bin_dirs[:]
  113. old_path = self._save_env["PATH"]
  114. if old_path:
  115. path.extend(old_path.split(os.pathsep))
  116. pythonpath = [self._site_dir]
  117. os.environ.update(
  118. {
  119. "PATH": os.pathsep.join(path),
  120. "PYTHONNOUSERSITE": "1",
  121. "PYTHONPATH": os.pathsep.join(pythonpath),
  122. }
  123. )
  124. def __exit__(
  125. self,
  126. exc_type: Optional[Type[BaseException]],
  127. exc_val: Optional[BaseException],
  128. exc_tb: Optional[TracebackType],
  129. ) -> None:
  130. for varname, old_value in self._save_env.items():
  131. if old_value is None:
  132. os.environ.pop(varname, None)
  133. else:
  134. os.environ[varname] = old_value
  135. def check_requirements(
  136. self, reqs: Iterable[str]
  137. ) -> Tuple[Set[Tuple[str, str]], Set[str]]:
  138. """Return 2 sets:
  139. - conflicting requirements: set of (installed, wanted) reqs tuples
  140. - missing requirements: set of reqs
  141. """
  142. missing = set()
  143. conflicting = set()
  144. if reqs:
  145. env = get_environment(self._lib_dirs)
  146. for req_str in reqs:
  147. req = Requirement(req_str)
  148. dist = env.get_distribution(req.name)
  149. if not dist:
  150. missing.add(req_str)
  151. continue
  152. if isinstance(dist.version, Version):
  153. installed_req_str = f"{req.name}=={dist.version}"
  154. else:
  155. installed_req_str = f"{req.name}==={dist.version}"
  156. if dist.version not in req.specifier:
  157. conflicting.add((installed_req_str, req_str))
  158. # FIXME: Consider direct URL?
  159. return conflicting, missing
  160. def install_requirements(
  161. self,
  162. finder: "PackageFinder",
  163. requirements: Iterable[str],
  164. prefix_as_string: str,
  165. *,
  166. kind: str,
  167. ) -> None:
  168. prefix = self._prefixes[prefix_as_string]
  169. assert not prefix.setup
  170. prefix.setup = True
  171. if not requirements:
  172. return
  173. with contextlib.ExitStack() as ctx:
  174. pip_runnable = ctx.enter_context(_create_standalone_pip())
  175. self._install_requirements(
  176. pip_runnable,
  177. finder,
  178. requirements,
  179. prefix,
  180. kind=kind,
  181. )
  182. @staticmethod
  183. def _install_requirements(
  184. pip_runnable: str,
  185. finder: "PackageFinder",
  186. requirements: Iterable[str],
  187. prefix: _Prefix,
  188. *,
  189. kind: str,
  190. ) -> None:
  191. args: List[str] = [
  192. sys.executable,
  193. pip_runnable,
  194. "install",
  195. "--ignore-installed",
  196. "--no-user",
  197. "--prefix",
  198. prefix.path,
  199. "--no-warn-script-location",
  200. ]
  201. if logger.getEffectiveLevel() <= logging.DEBUG:
  202. args.append("-v")
  203. for format_control in ("no_binary", "only_binary"):
  204. formats = getattr(finder.format_control, format_control)
  205. args.extend(
  206. (
  207. "--" + format_control.replace("_", "-"),
  208. ",".join(sorted(formats or {":none:"})),
  209. )
  210. )
  211. index_urls = finder.index_urls
  212. if index_urls:
  213. args.extend(["-i", index_urls[0]])
  214. for extra_index in index_urls[1:]:
  215. args.extend(["--extra-index-url", extra_index])
  216. else:
  217. args.append("--no-index")
  218. for link in finder.find_links:
  219. args.extend(["--find-links", link])
  220. for host in finder.trusted_hosts:
  221. args.extend(["--trusted-host", host])
  222. if finder.allow_all_prereleases:
  223. args.append("--pre")
  224. if finder.prefer_binary:
  225. args.append("--prefer-binary")
  226. args.append("--")
  227. args.extend(requirements)
  228. extra_environ = {"_PIP_STANDALONE_CERT": where()}
  229. with open_spinner(f"Installing {kind}") as spinner:
  230. call_subprocess(
  231. args,
  232. command_desc=f"pip subprocess to install {kind}",
  233. spinner=spinner,
  234. extra_environ=extra_environ,
  235. )
  236. class NoOpBuildEnvironment(BuildEnvironment):
  237. """A no-op drop-in replacement for BuildEnvironment"""
  238. def __init__(self) -> None:
  239. pass
  240. def __enter__(self) -> None:
  241. pass
  242. def __exit__(
  243. self,
  244. exc_type: Optional[Type[BaseException]],
  245. exc_val: Optional[BaseException],
  246. exc_tb: Optional[TracebackType],
  247. ) -> None:
  248. pass
  249. def cleanup(self) -> None:
  250. pass
  251. def install_requirements(
  252. self,
  253. finder: "PackageFinder",
  254. requirements: Iterable[str],
  255. prefix_as_string: str,
  256. *,
  257. kind: str,
  258. ) -> None:
  259. raise NotImplementedError()