Skip to content

Commit

Permalink
Be more flexible on accepted paths.
Browse files Browse the repository at this point in the history
  • Loading branch information
felixfontein committed Oct 5, 2023
1 parent 0d1d8d5 commit d1d6edb
Show file tree
Hide file tree
Showing 8 changed files with 86 additions and 43 deletions.
27 changes: 17 additions & 10 deletions src/antsibull_core/ansible_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

if t.TYPE_CHECKING:
import aiohttp.client
from _typeshed import StrPath


mlog = log.fields(mod=__name__)
Expand Down Expand Up @@ -133,7 +134,7 @@ async def get_latest_version(self) -> PypiVer:
versions = await self.get_versions()
return versions[0]

async def retrieve(self, ansible_core_version: str, download_dir: str) -> str:
async def retrieve(self, ansible_core_version: str, download_dir: StrPath) -> str:
"""
Get the release from pypi.
Expand Down Expand Up @@ -205,7 +206,7 @@ def get_ansible_core_package_name(ansible_core_version: str | PypiVer) -> str:
return "ansible-core"


def _get_source_version(ansible_core_source: str) -> PypiVer:
def _get_source_version(ansible_core_source: StrPath) -> PypiVer:
with open(
os.path.join(ansible_core_source, "lib", "ansible", "release.py"),
"r",
Expand Down Expand Up @@ -238,7 +239,7 @@ def _version_is_devel(version: PypiVer) -> bool:
return bool(dev_version.match(version.public))


def source_is_devel(ansible_core_source: str | None) -> bool:
def source_is_devel(ansible_core_source: StrPath | None) -> bool:
"""
:arg ansible_core_source: A path to an Ansible-core checkout or expanded sdist or None.
This will be used instead of downloading an ansible-core package if the version matches
Expand All @@ -257,7 +258,7 @@ def source_is_devel(ansible_core_source: str | None) -> bool:


def source_is_correct_version(
ansible_core_source: str | None, ansible_core_version: PypiVer
ansible_core_source: StrPath | None, ansible_core_version: PypiVer
) -> bool:
"""
:arg ansible_core_source: A path to an Ansible-core checkout or expanded sdist or None.
Expand Down Expand Up @@ -286,7 +287,9 @@ def source_is_correct_version(
return False


async def checkout_from_git(download_dir: str, repo_url: t.Optional[str] = None) -> str:
async def checkout_from_git(
download_dir: StrPath, repo_url: t.Optional[str] = None
) -> str:
"""
Checkout the ansible-core git repo.
Expand All @@ -303,7 +306,7 @@ async def checkout_from_git(download_dir: str, repo_url: t.Optional[str] = None)
return ansible_core_dir


async def create_sdist(source_dir: str, dest_dir: str) -> str:
async def create_sdist(source_dir: StrPath, dest_dir: StrPath) -> str:
"""
Create an sdist for the python package at a given path.
Expand All @@ -313,7 +316,11 @@ async def create_sdist(source_dir: str, dest_dir: str) -> str:
"""

# Make a subdir of dest_dir for returning the dist in
dist_dir_prefix = os.path.join(os.path.basename(source_dir))
dist_dir_prefix = os.path.join(
os.path.basename(
source_dir # pyre-ignore[6]: basename() accepts path-like object
)
)
dist_dir = tempfile.mkdtemp(prefix=dist_dir_prefix, dir=dest_dir)

try:
Expand Down Expand Up @@ -342,8 +349,8 @@ async def create_sdist(source_dir: str, dest_dir: str) -> str:
async def get_ansible_core(
aio_session: "aiohttp.client.ClientSession",
ansible_core_version: str,
tmpdir: str,
ansible_core_source: str | None = None,
tmpdir: StrPath,
ansible_core_source: StrPath | None = None,
) -> str:
"""
Create an ansible-core directory of the requested version.
Expand All @@ -363,7 +370,7 @@ async def get_ansible_core(
if source_is_devel(ansible_core_source):
# source_is_devel() protects against this. This assert is to inform the type checker
assert ansible_core_source is not None
source_location: str = ansible_core_source
source_location: StrPath = ansible_core_source

else:
source_location = await checkout_from_git(tmpdir)
Expand Down
21 changes: 15 additions & 6 deletions src/antsibull_core/collections.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,35 +9,42 @@

import asyncio
import os
import typing as t

from .subprocess_util import async_log_run

if t.TYPE_CHECKING:
from _typeshed import StrPath


class CollectionFormatError(Exception):
pass


async def install_together(
collection_tarballs: list[str], ansible_collections_dir: str
collection_tarballs: list[StrPath], ansible_collections_dir: StrPath
) -> None:
installers = []
for pathname in collection_tarballs:
namespace, collection, _dummy = os.path.basename(pathname).split("-", 2)
basename = os.path.basename(
pathname # pyre-ignore[6]: basename() accepts path-like object
)
namespace, collection, _dummy = basename.split("-", 2)
collection_dir = os.path.join(ansible_collections_dir, namespace, collection)
# Note: mkdir -p equivalent is okay because we created package_dir ourselves as a directory
# that only we can access
os.makedirs(collection_dir, mode=0o700, exist_ok=False)
installers.append(
asyncio.create_task(
async_log_run(["tar", "-xf", pathname, "-C", collection_dir])
async_log_run(["tar", "-xf", f"{pathname}", "-C", collection_dir])
)
)

await asyncio.gather(*installers)


async def install_separately(
collection_tarballs: list[str], collection_dir: str
collection_tarballs: list[StrPath], collection_dir: StrPath
) -> list[str]:
installers = []
collection_dirs: list[str] = []
Expand All @@ -46,7 +53,9 @@ async def install_separately(
return collection_dirs

for pathname in collection_tarballs:
filename = os.path.basename(pathname)
filename = os.path.basename(
pathname # pyre-ignore[6]: basename() accepts path-like object
)
namespace, collection, version_ext = filename.split("-", 2)
version = None
for ext in (".tar.gz",):
Expand Down Expand Up @@ -76,7 +85,7 @@ async def install_separately(

installers.append(
asyncio.create_task(
async_log_run(["tar", "-xf", pathname, "-C", collection_dir])
async_log_run(["tar", "-xf", f"{pathname}", "-C", collection_dir])
)
)

Expand Down
22 changes: 16 additions & 6 deletions src/antsibull_core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from __future__ import annotations

import os.path
import typing as t
from collections.abc import Iterable, Mapping

import perky # type: ignore[import]
Expand All @@ -16,6 +17,9 @@
from .logging import log
from .schemas.context import AppContext, LibContext

if t.TYPE_CHECKING:
from _typeshed import StrPath

mlog = log.fields(mod=__name__)

#: System config file location.
Expand All @@ -29,7 +33,7 @@ class ConfigError(Exception):
pass


def find_config_files(conf_files: Iterable[str]) -> list[str]:
def find_config_files(conf_files: Iterable[StrPath] | Iterable[str]) -> list[str]:
"""
Find all config files that exist.
Expand All @@ -39,21 +43,26 @@ def find_config_files(conf_files: Iterable[str]) -> list[str]:
flog = mlog.fields(func="find_config_file")
flog.fields(conf_files=conf_files).debug("Enter")

paths = [os.path.abspath(p) for p in conf_files]
paths = [
os.path.abspath(p) # pyre-ignore[6]: abspath() accepts path-like object
for p in conf_files
]
flog.fields(paths=paths).info("Paths to check")

config_files = []
for conf_path in paths:
if os.path.exists(conf_path):
config_files.append(conf_path)
config_files.append(f"{conf_path}")
flog.fields(paths=config_files).info("Paths found")

flog.debug("Leave")
return config_files


def validate_config(
config: Mapping, filenames: list[str], app_context_model: type[AppContext]
config: Mapping,
filenames: list[StrPath] | list[str],
app_context_model: type[AppContext],
) -> None:
"""
Validate configuration.
Expand All @@ -76,12 +85,13 @@ def validate_config(
LibContext.parse_obj(lib)
app_context_model.parse_obj(app)
except p.ValidationError as exc:
joined_filenames = ", ".join(f"{fn}" for fn in filenames)
raise ConfigError(
f"Error while parsing configuration from {', '.join(filenames)}:\n{exc}"
f"Error while parsing configuration from {joined_filenames}:\n{exc}"
) from exc


def _load_config_file(filename: str) -> Mapping:
def _load_config_file(filename: StrPath) -> Mapping:
"""
Load configuration from one file and return the raw data.
"""
Expand Down
21 changes: 11 additions & 10 deletions src/antsibull_core/dependency_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from packaging.version import Version as PypiVer

if t.TYPE_CHECKING:
from _typeshed import StrOrBytesPath
from semantic_version import Version as SemVer


Expand All @@ -35,7 +36,7 @@ class InvalidFileFormat(Exception):
pass


def parse_pieces_file(pieces_file: str) -> list[str]:
def parse_pieces_file(pieces_file: StrOrBytesPath) -> list[str]:
with open(pieces_file, "rb") as f:
contents = f.read()

Expand All @@ -49,7 +50,7 @@ def parse_pieces_file(pieces_file: str) -> list[str]:
return collections


def _parse_name_version_spec_file(filename: str) -> DependencyFileData:
def _parse_name_version_spec_file(filename: StrOrBytesPath) -> DependencyFileData:
deps: dict[str, str] = {}
ansible_core_version: str | None = None
ansible_version: str | None = None
Expand All @@ -60,7 +61,7 @@ def _parse_name_version_spec_file(filename: str) -> DependencyFileData:
if record[0] in ("_ansible_version", "_acd_version"):
if ansible_version is not None:
raise InvalidFileFormat(
f"{filename} specified _ansible_version/_acd_version"
f"{filename!r} specified _ansible_version/_acd_version"
" more than once"
)
ansible_version = record[1]
Expand All @@ -69,7 +70,7 @@ def _parse_name_version_spec_file(filename: str) -> DependencyFileData:
if record[0] in ("_ansible_base_version", "_ansible_core_version"):
if ansible_core_version is not None:
raise InvalidFileFormat(
f"{filename} specified _ansible_base_version/_ansible_core_version more than"
f"{filename!r} specified _ansible_base_version/_ansible_core_version more than"
" once"
)
ansible_core_version = record[1]
Expand All @@ -79,12 +80,12 @@ def _parse_name_version_spec_file(filename: str) -> DependencyFileData:

if ansible_core_version is None:
raise InvalidFileFormat(
f"{filename} was invalid. It did not contain"
f"{filename!r} was invalid. It did not contain"
" the required ansible_core_version field"
)
if ansible_version is None:
raise InvalidFileFormat(
f"{filename} was invalid. It did not contain"
f"{filename!r} was invalid. It did not contain"
" the required ansible_version field"
)

Expand All @@ -111,13 +112,13 @@ class DepsFile:
ansible-core version, not an exact dependency on that precise version.
"""

def __init__(self, deps_file: str) -> None:
def __init__(self, deps_file: StrOrBytesPath) -> None:
"""
Create a :mod:`DepsFile`.
:arg deps_file: filename of the `DepsFile`.
"""
self.filename: str = deps_file
self.filename: StrOrBytesPath = deps_file

def parse(self) -> DependencyFileData:
"""Parse the deps from a dependency file."""
Expand Down Expand Up @@ -164,8 +165,8 @@ def write(


class BuildFile:
def __init__(self, build_file: str) -> None:
self.filename: str = build_file
def __init__(self, build_file: StrOrBytesPath) -> None:
self.filename: StrOrBytesPath = build_file

def parse(self) -> DependencyFileData:
"""Parse the build from a dependency file."""
Expand Down
13 changes: 9 additions & 4 deletions src/antsibull_core/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

from __future__ import annotations

import typing as t

from antsibull_core.subprocess_util import log_run

try:
Expand All @@ -15,6 +17,9 @@
except ImportError:
posix1e = None

if t.TYPE_CHECKING:
from _typeshed import StrPath


class UnableToCheck(Exception):
"""No library or command support to check this."""
Expand All @@ -24,15 +29,15 @@ class CheckFailure(Exception):
"""We checking failed unexpectedly."""


def _get_acls(path: str) -> str:
def _get_acls(path: StrPath) -> str:
"""Return the acls for a given file or raise an exception."""
acls = None
if posix1e:
acl = posix1e.ACL(file=path)
acl = posix1e.ACL(file=f"{path}")
acls = acl.to_any_text(options=posix1e.TEXT_NUMERIC_IDS).decode("utf-8")
else:
try:
acls = log_run(["getfacl", path, "-n"]).stdout
acls = log_run(["getfacl", f"{path}", "-n"]).stdout
except FileNotFoundError:
pass
except Exception as e:
Expand All @@ -45,7 +50,7 @@ def _get_acls(path: str) -> str:
return acls


def writable_via_acls(path: str, euid: int) -> bool:
def writable_via_acls(path: StrPath, euid: int) -> bool:
"""
Check whether acls gives someone other than the user write access to a path.
Expand Down
3 changes: 2 additions & 1 deletion src/antsibull_core/galaxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
# The type checker can handle finding aiohttp.client but flake8 cannot :-(
if t.TYPE_CHECKING:
import aiohttp.client
from _typeshed import StrPath


class NoSuchCollection(Exception):
Expand Down Expand Up @@ -380,7 +381,7 @@ class CollectionDownloader(GalaxyClient):
def __init__(
self,
aio_session: aiohttp.client.ClientSession,
download_dir: str,
download_dir: StrPath,
galaxy_server: t.Optional[str] = None,
collection_cache: str | None = None,
context: t.Optional[GalaxyContext] = None,
Expand Down
Loading

0 comments on commit d1d6edb

Please sign in to comment.