from pathlib import Path
from typing import Callable, Iterable, Iterator, List
from prospector.exceptions import PermissionMissing
from prospector.pathutils import is_python_module, is_python_package, is_virtualenv
_SKIP_DIRECTORIES = (".git", ".tox", ".mypy_cache", ".pytest_cache", ".venv", "__pycache__", "node_modules")
class FileFinder:
"""
This class is responsible for taking a combination of command-line arguments
and configuration loaded from a profile to discover all files and modules which
should be inspected.
Individual tools can be told to ignore certain files, so the job of this class
is basically to know which files to pass to which tools to be inspected.
"""
def __init__(self, *provided_paths: Path, exclusion_filters: Iterable[Callable] = None):
"""
:param provided_paths:
A list of Path objects to search for files and modules - can be either directories or files
:param exclusion_filters:
An optional list of filters. All paths will be checked against this list - if any return True,
the path is excluded.
"""
self._provided_files = []
self._provided_dirs = []
self._exclusion_filters = [
# we always want to ignore some things
lambda _path: _path.is_dir() and _path.name in _SKIP_DIRECTORIES,
is_virtualenv,
] + list(exclusion_filters or [])
for path in provided_paths:
if not path.exists():
raise FileNotFoundError(path)
# ensure all paths from now one are absolute paths; they can be converted
# to relative paths for output purposes later
path = path.absolute()
if path.is_file():
self._provided_files.append(path)
if path.is_dir():
self._provided_dirs.append(path)
def make_syspath(self) -> List[Path]:
paths = set()
for path in self._provided_dirs:
paths.add(path)
for module in self.python_modules:
paths.add(module.parent)
return sorted(paths)
def is_excluded(self, path: Path) -> bool:
return any(filt(path) for filt in self._exclusion_filters)
def _filter(self, paths: Iterable[Path]) -> List[Path]:
return [path for path in paths if not self.is_excluded(path)]
def _walk(self, directory: Path) -> Iterator[Path]:
if not self.is_excluded(directory):
yield directory
for path in directory.iterdir():
if self.is_excluded(path):
continue
if path.is_dir():
yield from self._walk(path)
else:
yield path
@property
def files(self) -> List[Path]:
"""
List every individual file found from the given configuration.
This method is useful for tools which require an explicit list of files to check.
"""
files = set()
for path in self._provided_files:
files.add(path)
for directory in self.directories:
for path in self._walk(directory):
if path.is_file():
files.add(path)
return self._filter(files)
@property
def python_packages(self) -> List[Path]:
"""
Lists every directory found in the given configuration which is a python module (that is,
contains an `__init__.py` file).
This method is useful for passing to tools which will do their own discovery of python files.
"""
return self._filter(d for d in self.directories if is_python_package(d))
@property
def python_modules(self) -> List[Path]:
"""
Lists every directory found in the given configuration which is a python module (that is,
contains an `__init__.py` file).
This method is useful for passing to tools which will do their own discovery of python files.
"""
return self._filter(f for f in self.files if is_python_module(f))
@property
def directories(self) -> List[Path]:
"""
Lists every directory found from the given configuration, regardless of its contents.
This method is useful for passing to tools which will do their own discovery of python files.
"""
dirs = set()
for directory in self._provided_dirs:
dirs.add(directory)
try:
for obj in self._walk(directory):
if obj.is_dir():
dirs.add(obj)
except PermissionError as err:
raise PermissionMissing(obj) from err
return self._filter(dirs)
|