"""
We acquire the python information by running an interrogation script via subprocess trigger. This operation is not
cheap, especially not on Windows. To not have to pay this hefty cost every time we apply multiple levels of
caching.
"""
import logging
import os
import random
import sys
from collections import OrderedDict
from pathlib import Path
from shlex import quote
from string import ascii_lowercase, ascii_uppercase, digits
from subprocess import Popen
from virtualenv.app_data import AppDataDisabled
from virtualenv.discovery.py_info import PythonInfo
from virtualenv.util.subprocess import subprocess
_CACHE = OrderedDict()
_CACHE[Path(sys.executable)] = PythonInfo()
def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=False):
env = os.environ if env is None else env
result = _get_from_cache(cls, app_data, exe, env, ignore_cache=ignore_cache)
if isinstance(result, Exception):
if raise_on_error:
raise result
else:
logging.info("%s", result)
result = None
return result
def _get_from_cache(cls, app_data, exe, env, ignore_cache=True):
# note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a
# pyenv.cfg somewhere alongside on python3.5+
exe_path = Path(exe)
if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache
result = _CACHE[exe_path]
else: # otherwise go through the app data cache
py_info = _get_via_file_cache(cls, app_data, exe_path, exe, env)
result = _CACHE[exe_path] = py_info
# independent if it was from the file or in-memory cache fix the original executable location
if isinstance(result, PythonInfo):
result.executable = exe
return result
def _get_via_file_cache(cls, app_data, path, exe, env):
path_text = str(path)
try:
path_modified = path.stat().st_mtime
except OSError:
path_modified = -1
if app_data is None:
app_data = AppDataDisabled()
py_info, py_info_store = None, app_data.py_info(path)
with py_info_store.locked():
if py_info_store.exists(): # if exists and matches load
data = py_info_store.read()
of_path, of_st_mtime, of_content = data["path"], data["st_mtime"], data["content"]
if of_path == path_text and of_st_mtime == path_modified:
py_info = cls._from_dict(of_content.copy())
sys_exe = py_info.system_executable
if sys_exe is not None and not os.path.exists(sys_exe):
py_info_store.remove()
py_info = None
else:
py_info_store.remove()
if py_info is None: # if not loaded run and save
failure, py_info = _run_subprocess(cls, exe, app_data, env)
if failure is None:
data = {"st_mtime": path_modified, "path": path_text, "content": py_info._to_dict()}
py_info_store.write(data)
else:
py_info = failure
return py_info
COOKIE_LENGTH: int = 32
def gen_cookie():
return "".join(random.choice("".join((ascii_lowercase, ascii_uppercase, digits))) for _ in range(COOKIE_LENGTH))
def _run_subprocess(cls, exe, app_data, env):
py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py"
# Cookies allow to split the serialized stdout output generated by the script collecting the info from the output
# generated by something else. The right way to deal with it is to create an anonymous pipe and pass its descriptor
# to the child and output to it. But AFAIK all of them are either not cross-platform or too big to implement and are
# not in the stdlib. So the easiest and the shortest way I could mind is just using the cookies.
# We generate pseudorandom cookies because it easy to implement and avoids breakage from outputting modules source
# code, i.e. by debug output libraries. We reverse the cookies to avoid breakages resulting from variable values
# appearing in debug output.
start_cookie = gen_cookie()
end_cookie = gen_cookie()
with app_data.ensure_extracted(py_info_script) as py_info_script:
cmd = [exe, str(py_info_script), start_cookie, end_cookie]
# prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490
env = env.copy()
env.pop("__PYVENV_LAUNCHER__", None)
logging.debug("get interpreter info via cmd: %s", LogCmd(cmd))
try:
process = Popen(
cmd,
universal_newlines=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=env,
encoding="utf-8",
)
out, err = process.communicate()
code = process.returncode
except OSError as os_error:
out, err, code = "", os_error.strerror, os_error.errno
result, failure = None, None
if code == 0:
out_starts = out.find(start_cookie[::-1])
if out_starts > -1:
pre_cookie = out[:out_starts]
if pre_cookie:
sys.stdout.write(pre_cookie)
out = out[out_starts + COOKIE_LENGTH :]
out_ends = out.find(end_cookie[::-1])
if out_ends > -1:
post_cookie = out[out_ends + COOKIE_LENGTH :]
if post_cookie:
sys.stdout.write(post_cookie)
out = out[:out_ends]
result = cls._from_json(out)
result.executable = exe # keep original executable as this may contain initialization code
else:
msg = f"{exe} with code {code}{f' out: {out!r}' if out else ''}{f' err: {err!r}' if err else ''}"
failure = RuntimeError(f"failed to query {msg}")
return failure, result
class LogCmd:
def __init__(self, cmd, env=None):
self.cmd = cmd
self.env = env
def __repr__(self):
cmd_repr = " ".join(quote(str(c)) for c in self.cmd)
if self.env is not None:
cmd_repr = f"{cmd_repr} env of {self.env!r}"
return cmd_repr
def clear(app_data):
app_data.py_info_clear()
_CACHE.clear()
___all___ = [
"from_exe",
"clear",
"LogCmd",
]
|