build_env.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283
  1. """Build Environment used for isolation during sdist building
  2. """
  3. import contextlib
  4. import logging
  5. import os
  6. import pathlib
  7. import sys
  8. import textwrap
  9. import zipfile
  10. from collections import OrderedDict
  11. from sysconfig import get_paths
  12. from types import TracebackType
  13. from typing import TYPE_CHECKING, Iterable, Iterator, List, Optional, Set, Tuple, Type
  14. from pip._vendor.certifi import where
  15. from pip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet
  16. from pip import __file__ as pip_location
  17. from pip._internal.cli.spinners import open_spinner
  18. from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
  19. from pip._internal.utils.subprocess import call_subprocess
  20. from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
  21. if TYPE_CHECKING:
  22. from pip._internal.index.package_finder import PackageFinder
  23. logger = logging.getLogger(__name__)
  24. class _Prefix:
  25. def __init__(self, path):
  26. # type: (str) -> None
  27. self.path = path
  28. self.setup = False
  29. self.bin_dir = get_paths(
  30. 'nt' if os.name == 'nt' else 'posix_prefix',
  31. vars={'base': path, 'platbase': path}
  32. )['scripts']
  33. self.lib_dirs = get_prefixed_libs(path)
  34. @contextlib.contextmanager
  35. def _create_standalone_pip() -> Iterator[str]:
  36. """Create a "standalone pip" zip file.
  37. The zip file's content is identical to the currently-running pip.
  38. It will be used to install requirements into the build environment.
  39. """
  40. source = pathlib.Path(pip_location).resolve().parent
  41. # Return the current instance if it is already a zip file. This can happen
  42. # if a PEP 517 requirement is an sdist itself.
  43. if not source.is_dir() and source.parent.name == "__env_pip__.zip":
  44. yield str(source)
  45. return
  46. with TempDirectory(kind="standalone-pip") as tmp_dir:
  47. pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip")
  48. with zipfile.ZipFile(pip_zip, "w") as zf:
  49. for child in source.rglob("*"):
  50. zf.write(child, child.relative_to(source.parent).as_posix())
  51. yield os.path.join(pip_zip, "pip")
  52. class BuildEnvironment:
  53. """Creates and manages an isolated environment to install build deps
  54. """
  55. def __init__(self):
  56. # type: () -> None
  57. temp_dir = TempDirectory(
  58. kind=tempdir_kinds.BUILD_ENV, globally_managed=True
  59. )
  60. self._prefixes = OrderedDict(
  61. (name, _Prefix(os.path.join(temp_dir.path, name)))
  62. for name in ('normal', 'overlay')
  63. )
  64. self._bin_dirs = [] # type: List[str]
  65. self._lib_dirs = [] # type: List[str]
  66. for prefix in reversed(list(self._prefixes.values())):
  67. self._bin_dirs.append(prefix.bin_dir)
  68. self._lib_dirs.extend(prefix.lib_dirs)
  69. # Customize site to:
  70. # - ensure .pth files are honored
  71. # - prevent access to system site packages
  72. system_sites = {
  73. os.path.normcase(site) for site in (get_purelib(), get_platlib())
  74. }
  75. self._site_dir = os.path.join(temp_dir.path, 'site')
  76. if not os.path.exists(self._site_dir):
  77. os.mkdir(self._site_dir)
  78. with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp:
  79. fp.write(textwrap.dedent(
  80. '''
  81. import os, site, sys
  82. # First, drop system-sites related paths.
  83. original_sys_path = sys.path[:]
  84. known_paths = set()
  85. for path in {system_sites!r}:
  86. site.addsitedir(path, known_paths=known_paths)
  87. system_paths = set(
  88. os.path.normcase(path)
  89. for path in sys.path[len(original_sys_path):]
  90. )
  91. original_sys_path = [
  92. path for path in original_sys_path
  93. if os.path.normcase(path) not in system_paths
  94. ]
  95. sys.path = original_sys_path
  96. # Second, add lib directories.
  97. # ensuring .pth file are processed.
  98. for path in {lib_dirs!r}:
  99. assert not path in sys.path
  100. site.addsitedir(path)
  101. '''
  102. ).format(system_sites=system_sites, lib_dirs=self._lib_dirs))
  103. def __enter__(self):
  104. # type: () -> None
  105. self._save_env = {
  106. name: os.environ.get(name, None)
  107. for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH')
  108. }
  109. path = self._bin_dirs[:]
  110. old_path = self._save_env['PATH']
  111. if old_path:
  112. path.extend(old_path.split(os.pathsep))
  113. pythonpath = [self._site_dir]
  114. os.environ.update({
  115. 'PATH': os.pathsep.join(path),
  116. 'PYTHONNOUSERSITE': '1',
  117. 'PYTHONPATH': os.pathsep.join(pythonpath),
  118. })
  119. def __exit__(
  120. self,
  121. exc_type, # type: Optional[Type[BaseException]]
  122. exc_val, # type: Optional[BaseException]
  123. exc_tb # type: Optional[TracebackType]
  124. ):
  125. # type: (...) -> None
  126. for varname, old_value in self._save_env.items():
  127. if old_value is None:
  128. os.environ.pop(varname, None)
  129. else:
  130. os.environ[varname] = old_value
  131. def check_requirements(self, reqs):
  132. # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
  133. """Return 2 sets:
  134. - conflicting requirements: set of (installed, wanted) reqs tuples
  135. - missing requirements: set of reqs
  136. """
  137. missing = set()
  138. conflicting = set()
  139. if reqs:
  140. ws = WorkingSet(self._lib_dirs)
  141. for req in reqs:
  142. try:
  143. if ws.find(Requirement.parse(req)) is None:
  144. missing.add(req)
  145. except VersionConflict as e:
  146. conflicting.add((str(e.args[0].as_requirement()),
  147. str(e.args[1])))
  148. return conflicting, missing
  149. def install_requirements(
  150. self,
  151. finder, # type: PackageFinder
  152. requirements, # type: Iterable[str]
  153. prefix_as_string, # type: str
  154. message # type: str
  155. ):
  156. # type: (...) -> None
  157. prefix = self._prefixes[prefix_as_string]
  158. assert not prefix.setup
  159. prefix.setup = True
  160. if not requirements:
  161. return
  162. with contextlib.ExitStack() as ctx:
  163. # TODO: Remove this block when dropping 3.6 support. Python 3.6
  164. # lacks importlib.resources and pep517 has issues loading files in
  165. # a zip, so we fallback to the "old" method by adding the current
  166. # pip directory to the child process's sys.path.
  167. if sys.version_info < (3, 7):
  168. pip_runnable = os.path.dirname(pip_location)
  169. else:
  170. pip_runnable = ctx.enter_context(_create_standalone_pip())
  171. self._install_requirements(
  172. pip_runnable,
  173. finder,
  174. requirements,
  175. prefix,
  176. message,
  177. )
  178. @staticmethod
  179. def _install_requirements(
  180. pip_runnable: str,
  181. finder: "PackageFinder",
  182. requirements: Iterable[str],
  183. prefix: _Prefix,
  184. message: str,
  185. ) -> None:
  186. args = [
  187. sys.executable, pip_runnable, 'install',
  188. '--ignore-installed', '--no-user', '--prefix', prefix.path,
  189. '--no-warn-script-location',
  190. ] # type: List[str]
  191. if logger.getEffectiveLevel() <= logging.DEBUG:
  192. args.append('-v')
  193. for format_control in ('no_binary', 'only_binary'):
  194. formats = getattr(finder.format_control, format_control)
  195. args.extend(('--' + format_control.replace('_', '-'),
  196. ','.join(sorted(formats or {':none:'}))))
  197. index_urls = finder.index_urls
  198. if index_urls:
  199. args.extend(['-i', index_urls[0]])
  200. for extra_index in index_urls[1:]:
  201. args.extend(['--extra-index-url', extra_index])
  202. else:
  203. args.append('--no-index')
  204. for link in finder.find_links:
  205. args.extend(['--find-links', link])
  206. for host in finder.trusted_hosts:
  207. args.extend(['--trusted-host', host])
  208. if finder.allow_all_prereleases:
  209. args.append('--pre')
  210. if finder.prefer_binary:
  211. args.append('--prefer-binary')
  212. args.append('--')
  213. args.extend(requirements)
  214. extra_environ = {"_PIP_STANDALONE_CERT": where()}
  215. with open_spinner(message) as spinner:
  216. call_subprocess(args, spinner=spinner, extra_environ=extra_environ)
  217. class NoOpBuildEnvironment(BuildEnvironment):
  218. """A no-op drop-in replacement for BuildEnvironment
  219. """
  220. def __init__(self):
  221. # type: () -> None
  222. pass
  223. def __enter__(self):
  224. # type: () -> None
  225. pass
  226. def __exit__(
  227. self,
  228. exc_type, # type: Optional[Type[BaseException]]
  229. exc_val, # type: Optional[BaseException]
  230. exc_tb # type: Optional[TracebackType]
  231. ):
  232. # type: (...) -> None
  233. pass
  234. def cleanup(self):
  235. # type: () -> None
  236. pass
  237. def install_requirements(
  238. self,
  239. finder, # type: PackageFinder
  240. requirements, # type: Iterable[str]
  241. prefix_as_string, # type: str
  242. message # type: str
  243. ):
  244. # type: (...) -> None
  245. raise NotImplementedError()