resolver.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462
  1. """Dependency Resolution
  2. The dependency resolution in pip is performed as follows:
  3. for top-level requirements:
  4. a. only one spec allowed per project, regardless of conflicts or not.
  5. otherwise a "double requirement" exception is raised
  6. b. they override sub-dependency requirements.
  7. for sub-dependencies
  8. a. "first found, wins" (where the order is breadth first)
  9. """
  10. # The following comment should be removed at some point in the future.
  11. # mypy: strict-optional=False
  12. import logging
  13. import sys
  14. from collections import defaultdict
  15. from itertools import chain
  16. from typing import DefaultDict, Iterable, List, Optional, Set, Tuple
  17. from pip._vendor.packaging import specifiers
  18. from pip._vendor.pkg_resources import Distribution
  19. from pip._internal.cache import WheelCache
  20. from pip._internal.exceptions import (
  21. BestVersionAlreadyInstalled,
  22. DistributionNotFound,
  23. HashError,
  24. HashErrors,
  25. UnsupportedPythonVersion,
  26. )
  27. from pip._internal.index.package_finder import PackageFinder
  28. from pip._internal.models.link import Link
  29. from pip._internal.operations.prepare import RequirementPreparer
  30. from pip._internal.req.req_install import (
  31. InstallRequirement,
  32. check_invalid_constraint_type,
  33. )
  34. from pip._internal.req.req_set import RequirementSet
  35. from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
  36. from pip._internal.utils.compatibility_tags import get_supported
  37. from pip._internal.utils.logging import indent_log
  38. from pip._internal.utils.misc import dist_in_usersite, normalize_version_info
  39. from pip._internal.utils.packaging import check_requires_python, get_requires_python
  40. logger = logging.getLogger(__name__)
  41. DiscoveredDependencies = DefaultDict[str, List[InstallRequirement]]
  42. def _check_dist_requires_python(
  43. dist, # type: Distribution
  44. version_info, # type: Tuple[int, int, int]
  45. ignore_requires_python=False, # type: bool
  46. ):
  47. # type: (...) -> None
  48. """
  49. Check whether the given Python version is compatible with a distribution's
  50. "Requires-Python" value.
  51. :param version_info: A 3-tuple of ints representing the Python
  52. major-minor-micro version to check.
  53. :param ignore_requires_python: Whether to ignore the "Requires-Python"
  54. value if the given Python version isn't compatible.
  55. :raises UnsupportedPythonVersion: When the given Python version isn't
  56. compatible.
  57. """
  58. requires_python = get_requires_python(dist)
  59. try:
  60. is_compatible = check_requires_python(
  61. requires_python, version_info=version_info
  62. )
  63. except specifiers.InvalidSpecifier as exc:
  64. logger.warning(
  65. "Package %r has an invalid Requires-Python: %s", dist.project_name, exc
  66. )
  67. return
  68. if is_compatible:
  69. return
  70. version = ".".join(map(str, version_info))
  71. if ignore_requires_python:
  72. logger.debug(
  73. "Ignoring failed Requires-Python check for package %r: " "%s not in %r",
  74. dist.project_name,
  75. version,
  76. requires_python,
  77. )
  78. return
  79. raise UnsupportedPythonVersion(
  80. "Package {!r} requires a different Python: {} not in {!r}".format(
  81. dist.project_name, version, requires_python
  82. )
  83. )
  84. class Resolver(BaseResolver):
  85. """Resolves which packages need to be installed/uninstalled to perform \
  86. the requested operation without breaking the requirements of any package.
  87. """
  88. _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"}
  89. def __init__(
  90. self,
  91. preparer, # type: RequirementPreparer
  92. finder, # type: PackageFinder
  93. wheel_cache, # type: Optional[WheelCache]
  94. make_install_req, # type: InstallRequirementProvider
  95. use_user_site, # type: bool
  96. ignore_dependencies, # type: bool
  97. ignore_installed, # type: bool
  98. ignore_requires_python, # type: bool
  99. force_reinstall, # type: bool
  100. upgrade_strategy, # type: str
  101. py_version_info=None, # type: Optional[Tuple[int, ...]]
  102. ):
  103. # type: (...) -> None
  104. super().__init__()
  105. assert upgrade_strategy in self._allowed_strategies
  106. if py_version_info is None:
  107. py_version_info = sys.version_info[:3]
  108. else:
  109. py_version_info = normalize_version_info(py_version_info)
  110. self._py_version_info = py_version_info
  111. self.preparer = preparer
  112. self.finder = finder
  113. self.wheel_cache = wheel_cache
  114. self.upgrade_strategy = upgrade_strategy
  115. self.force_reinstall = force_reinstall
  116. self.ignore_dependencies = ignore_dependencies
  117. self.ignore_installed = ignore_installed
  118. self.ignore_requires_python = ignore_requires_python
  119. self.use_user_site = use_user_site
  120. self._make_install_req = make_install_req
  121. self._discovered_dependencies = defaultdict(
  122. list
  123. ) # type: DiscoveredDependencies
  124. def resolve(self, root_reqs, check_supported_wheels):
  125. # type: (List[InstallRequirement], bool) -> RequirementSet
  126. """Resolve what operations need to be done
  127. As a side-effect of this method, the packages (and their dependencies)
  128. are downloaded, unpacked and prepared for installation. This
  129. preparation is done by ``pip.operations.prepare``.
  130. Once PyPI has static dependency metadata available, it would be
  131. possible to move the preparation to become a step separated from
  132. dependency resolution.
  133. """
  134. requirement_set = RequirementSet(check_supported_wheels=check_supported_wheels)
  135. for req in root_reqs:
  136. if req.constraint:
  137. check_invalid_constraint_type(req)
  138. requirement_set.add_requirement(req)
  139. # Actually prepare the files, and collect any exceptions. Most hash
  140. # exceptions cannot be checked ahead of time, because
  141. # _populate_link() needs to be called before we can make decisions
  142. # based on link type.
  143. discovered_reqs = [] # type: List[InstallRequirement]
  144. hash_errors = HashErrors()
  145. for req in chain(requirement_set.all_requirements, discovered_reqs):
  146. try:
  147. discovered_reqs.extend(self._resolve_one(requirement_set, req))
  148. except HashError as exc:
  149. exc.req = req
  150. hash_errors.append(exc)
  151. if hash_errors:
  152. raise hash_errors
  153. return requirement_set
  154. def _is_upgrade_allowed(self, req):
  155. # type: (InstallRequirement) -> bool
  156. if self.upgrade_strategy == "to-satisfy-only":
  157. return False
  158. elif self.upgrade_strategy == "eager":
  159. return True
  160. else:
  161. assert self.upgrade_strategy == "only-if-needed"
  162. return req.user_supplied or req.constraint
  163. def _set_req_to_reinstall(self, req):
  164. # type: (InstallRequirement) -> None
  165. """
  166. Set a requirement to be installed.
  167. """
  168. # Don't uninstall the conflict if doing a user install and the
  169. # conflict is not a user install.
  170. if not self.use_user_site or dist_in_usersite(req.satisfied_by):
  171. req.should_reinstall = True
  172. req.satisfied_by = None
  173. def _check_skip_installed(self, req_to_install):
  174. # type: (InstallRequirement) -> Optional[str]
  175. """Check if req_to_install should be skipped.
  176. This will check if the req is installed, and whether we should upgrade
  177. or reinstall it, taking into account all the relevant user options.
  178. After calling this req_to_install will only have satisfied_by set to
  179. None if the req_to_install is to be upgraded/reinstalled etc. Any
  180. other value will be a dist recording the current thing installed that
  181. satisfies the requirement.
  182. Note that for vcs urls and the like we can't assess skipping in this
  183. routine - we simply identify that we need to pull the thing down,
  184. then later on it is pulled down and introspected to assess upgrade/
  185. reinstalls etc.
  186. :return: A text reason for why it was skipped, or None.
  187. """
  188. if self.ignore_installed:
  189. return None
  190. req_to_install.check_if_exists(self.use_user_site)
  191. if not req_to_install.satisfied_by:
  192. return None
  193. if self.force_reinstall:
  194. self._set_req_to_reinstall(req_to_install)
  195. return None
  196. if not self._is_upgrade_allowed(req_to_install):
  197. if self.upgrade_strategy == "only-if-needed":
  198. return "already satisfied, skipping upgrade"
  199. return "already satisfied"
  200. # Check for the possibility of an upgrade. For link-based
  201. # requirements we have to pull the tree down and inspect to assess
  202. # the version #, so it's handled way down.
  203. if not req_to_install.link:
  204. try:
  205. self.finder.find_requirement(req_to_install, upgrade=True)
  206. except BestVersionAlreadyInstalled:
  207. # Then the best version is installed.
  208. return "already up-to-date"
  209. except DistributionNotFound:
  210. # No distribution found, so we squash the error. It will
  211. # be raised later when we re-try later to do the install.
  212. # Why don't we just raise here?
  213. pass
  214. self._set_req_to_reinstall(req_to_install)
  215. return None
  216. def _find_requirement_link(self, req):
  217. # type: (InstallRequirement) -> Optional[Link]
  218. upgrade = self._is_upgrade_allowed(req)
  219. best_candidate = self.finder.find_requirement(req, upgrade)
  220. if not best_candidate:
  221. return None
  222. # Log a warning per PEP 592 if necessary before returning.
  223. link = best_candidate.link
  224. if link.is_yanked:
  225. reason = link.yanked_reason or "<none given>"
  226. msg = (
  227. # Mark this as a unicode string to prevent
  228. # "UnicodeEncodeError: 'ascii' codec can't encode character"
  229. # in Python 2 when the reason contains non-ascii characters.
  230. "The candidate selected for download or install is a "
  231. "yanked version: {candidate}\n"
  232. "Reason for being yanked: {reason}"
  233. ).format(candidate=best_candidate, reason=reason)
  234. logger.warning(msg)
  235. return link
  236. def _populate_link(self, req):
  237. # type: (InstallRequirement) -> None
  238. """Ensure that if a link can be found for this, that it is found.
  239. Note that req.link may still be None - if the requirement is already
  240. installed and not needed to be upgraded based on the return value of
  241. _is_upgrade_allowed().
  242. If preparer.require_hashes is True, don't use the wheel cache, because
  243. cached wheels, always built locally, have different hashes than the
  244. files downloaded from the index server and thus throw false hash
  245. mismatches. Furthermore, cached wheels at present have undeterministic
  246. contents due to file modification times.
  247. """
  248. if req.link is None:
  249. req.link = self._find_requirement_link(req)
  250. if self.wheel_cache is None or self.preparer.require_hashes:
  251. return
  252. cache_entry = self.wheel_cache.get_cache_entry(
  253. link=req.link,
  254. package_name=req.name,
  255. supported_tags=get_supported(),
  256. )
  257. if cache_entry is not None:
  258. logger.debug("Using cached wheel link: %s", cache_entry.link)
  259. if req.link is req.original_link and cache_entry.persistent:
  260. req.original_link_is_in_wheel_cache = True
  261. req.link = cache_entry.link
  262. def _get_dist_for(self, req):
  263. # type: (InstallRequirement) -> Distribution
  264. """Takes a InstallRequirement and returns a single AbstractDist \
  265. representing a prepared variant of the same.
  266. """
  267. if req.editable:
  268. return self.preparer.prepare_editable_requirement(req)
  269. # satisfied_by is only evaluated by calling _check_skip_installed,
  270. # so it must be None here.
  271. assert req.satisfied_by is None
  272. skip_reason = self._check_skip_installed(req)
  273. if req.satisfied_by:
  274. return self.preparer.prepare_installed_requirement(req, skip_reason)
  275. # We eagerly populate the link, since that's our "legacy" behavior.
  276. self._populate_link(req)
  277. dist = self.preparer.prepare_linked_requirement(req)
  278. # NOTE
  279. # The following portion is for determining if a certain package is
  280. # going to be re-installed/upgraded or not and reporting to the user.
  281. # This should probably get cleaned up in a future refactor.
  282. # req.req is only avail after unpack for URL
  283. # pkgs repeat check_if_exists to uninstall-on-upgrade
  284. # (#14)
  285. if not self.ignore_installed:
  286. req.check_if_exists(self.use_user_site)
  287. if req.satisfied_by:
  288. should_modify = (
  289. self.upgrade_strategy != "to-satisfy-only"
  290. or self.force_reinstall
  291. or self.ignore_installed
  292. or req.link.scheme == "file"
  293. )
  294. if should_modify:
  295. self._set_req_to_reinstall(req)
  296. else:
  297. logger.info(
  298. "Requirement already satisfied (use --upgrade to upgrade):" " %s",
  299. req,
  300. )
  301. return dist
  302. def _resolve_one(
  303. self,
  304. requirement_set, # type: RequirementSet
  305. req_to_install, # type: InstallRequirement
  306. ):
  307. # type: (...) -> List[InstallRequirement]
  308. """Prepare a single requirements file.
  309. :return: A list of additional InstallRequirements to also install.
  310. """
  311. # Tell user what we are doing for this requirement:
  312. # obtain (editable), skipping, processing (local url), collecting
  313. # (remote url or package name)
  314. if req_to_install.constraint or req_to_install.prepared:
  315. return []
  316. req_to_install.prepared = True
  317. # Parse and return dependencies
  318. dist = self._get_dist_for(req_to_install)
  319. # This will raise UnsupportedPythonVersion if the given Python
  320. # version isn't compatible with the distribution's Requires-Python.
  321. _check_dist_requires_python(
  322. dist,
  323. version_info=self._py_version_info,
  324. ignore_requires_python=self.ignore_requires_python,
  325. )
  326. more_reqs = [] # type: List[InstallRequirement]
  327. def add_req(subreq, extras_requested):
  328. # type: (Distribution, Iterable[str]) -> None
  329. sub_install_req = self._make_install_req(
  330. str(subreq),
  331. req_to_install,
  332. )
  333. parent_req_name = req_to_install.name
  334. to_scan_again, add_to_parent = requirement_set.add_requirement(
  335. sub_install_req,
  336. parent_req_name=parent_req_name,
  337. extras_requested=extras_requested,
  338. )
  339. if parent_req_name and add_to_parent:
  340. self._discovered_dependencies[parent_req_name].append(add_to_parent)
  341. more_reqs.extend(to_scan_again)
  342. with indent_log():
  343. # We add req_to_install before its dependencies, so that we
  344. # can refer to it when adding dependencies.
  345. if not requirement_set.has_requirement(req_to_install.name):
  346. # 'unnamed' requirements will get added here
  347. # 'unnamed' requirements can only come from being directly
  348. # provided by the user.
  349. assert req_to_install.user_supplied
  350. requirement_set.add_requirement(req_to_install, parent_req_name=None)
  351. if not self.ignore_dependencies:
  352. if req_to_install.extras:
  353. logger.debug(
  354. "Installing extra requirements: %r",
  355. ",".join(req_to_install.extras),
  356. )
  357. missing_requested = sorted(
  358. set(req_to_install.extras) - set(dist.extras)
  359. )
  360. for missing in missing_requested:
  361. logger.warning("%s does not provide the extra '%s'", dist, missing)
  362. available_requested = sorted(
  363. set(dist.extras) & set(req_to_install.extras)
  364. )
  365. for subreq in dist.requires(available_requested):
  366. add_req(subreq, extras_requested=available_requested)
  367. return more_reqs
  368. def get_installation_order(self, req_set):
  369. # type: (RequirementSet) -> List[InstallRequirement]
  370. """Create the installation order.
  371. The installation order is topological - requirements are installed
  372. before the requiring thing. We break cycles at an arbitrary point,
  373. and make no other guarantees.
  374. """
  375. # The current implementation, which we may change at any point
  376. # installs the user specified things in the order given, except when
  377. # dependencies must come earlier to achieve topological order.
  378. order = []
  379. ordered_reqs = set() # type: Set[InstallRequirement]
  380. def schedule(req):
  381. # type: (InstallRequirement) -> None
  382. if req.satisfied_by or req in ordered_reqs:
  383. return
  384. if req.constraint:
  385. return
  386. ordered_reqs.add(req)
  387. for dep in self._discovered_dependencies[req.name]:
  388. schedule(dep)
  389. order.append(req)
  390. for install_req in req_set.requirements.values():
  391. schedule(install_req)
  392. return order