Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove the file dependency allowlist for gomod #1010

Merged
merged 3 commits into from
Jun 19, 2024
Merged

Remove the file dependency allowlist for gomod #1010

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d03c4fb
remove the file dependency allowlist for gomod
taylormadore Jun 12, 2024
54cee5f
fix resolve_gomod function typing
taylormadore Jun 18, 2024
7c8861c
refactor gomod vetting of local file dependencies
taylormadore Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions cachito/workers/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ class Config(object):
cachito_nexus_username = "cachito"
cachito_npm_file_deps_allowlist: Dict[str, List[str]] = {}
cachito_yarn_file_deps_allowlist: Dict[str, List[str]] = {}
cachito_gomod_file_deps_allowlist: Dict[str, List[str]] = {}
cachito_rubygems_file_deps_allowlist: Dict[str, List[str]] = {}
cachito_request_file_logs_dir: Optional[str] = None
cachito_request_file_logs_format = (
Expand Down Expand Up @@ -136,15 +135,6 @@ class DevelopmentConfig(Config):
cachito_nexus_url = "http://nexus:8081"
cachito_npm_file_deps_allowlist = {"cachito-npm-test": ["subpackage"]}
cachito_yarn_file_deps_allowlist = {"cachito-yarn-test": ["subpackage"]}
cachito_gomod_file_deps_allowlist = {
"github.com/cachito-testing/cachito-gomod-local-deps": ["github.com/cachito-testing/*"],
"github.com/cachito-testing/cachito-gomod-local-parent-deps/foo-module": [
"github.com/cachito-testing/*"
],
"github.com/cachito-testing/cachito-gomod-local-parent-deps/foo-module/bar-module": [
"github.com/cachito-testing/*"
],
}
cachito_rubygems_file_deps_allowlist = {
"cachito-rubygems-with-dependencies": ["pathgem"],
"cachito-rubygems-multiple/first_pkg": ["pathgem"],
Expand Down
112 changes: 24 additions & 88 deletions cachito/workers/pkg_managers/gomod.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import fnmatch
import functools
import logging
import os
Expand Down Expand Up @@ -281,7 +280,12 @@ def match_parent_module(package_name: str, module_names: Iterable[str]) -> Optio


@tracer.start_as_current_span("resolve_gomod")
def resolve_gomod(app_source_path, request, dep_replacements=None, git_dir_path=None):
def resolve_gomod(
app_source_path: Path,
request: dict[str, Any],
dep_replacements: Optional[list[dict[str, str]]] = None,
git_dir_path: Optional[Path] = None,
) -> dict[str, Any]:
"""
Resolve and fetch gomod dependencies for given app source archive.

Expand Down Expand Up @@ -455,17 +459,11 @@ def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[GoPackage]:
}
main_packages.append({"pkg": main_pkg, "pkg_deps": pkg_deps})

allowlist = _get_allowed_local_deps(main_module_name)
log.debug("Allowed local dependencies for %s: %s", main_module_name, allowlist)
_vet_local_deps(
main_module_deps, main_module_name, allowlist, app_source_path, git_dir_path
)
for pkg in main_packages:
_vet_local_file_dep_paths(main_module_deps, app_source_path, git_dir_path)
for package in main_packages:
# Local dependencies are always relative to the main module, even for subpackages
_vet_local_deps(
pkg["pkg_deps"], main_module_name, allowlist, app_source_path, git_dir_path
)
_set_full_local_dep_relpaths(pkg["pkg_deps"], main_module_deps)
_vet_local_file_dep_paths(package["pkg_deps"], app_source_path, git_dir_path)
_set_full_local_dep_relpaths(package["pkg_deps"], main_module_deps)

return {
"module": main_module,
Expand Down Expand Up @@ -510,7 +508,7 @@ def _get_name_and_version(module: GoModule) -> tuple[str, str]:
return name, version


def _should_vendor_deps(flags: List[str], app_dir: str, strict: bool) -> Tuple[bool, bool]:
def _should_vendor_deps(flags: List[str], app_dir: Path, strict: bool) -> Tuple[bool, bool]:
"""
Determine if Cachito should vendor dependencies and if it is allowed to make changes.

Expand All @@ -524,7 +522,7 @@ def _should_vendor_deps(flags: List[str], app_dir: str, strict: bool) -> Tuple[b
:return: (should vendor: bool, allowed to make changes in the vendor directory: bool)
:raise ValidationError: if the vendor dir is present, the flags are not used and we are strict
"""
vendor = Path(app_dir) / "vendor"
vendor = app_dir / "vendor"

if "gomod-vendor-check" in flags:
return True, not vendor.exists()
Expand All @@ -541,7 +539,7 @@ def _should_vendor_deps(flags: List[str], app_dir: str, strict: bool) -> Tuple[b


@tracer.start_as_current_span("_vendor_deps")
def _vendor_deps(go: Go, run_params: dict, can_make_changes: bool, git_dir: str) -> list[GoModule]:
def _vendor_deps(go: Go, run_params: dict, can_make_changes: bool, git_dir: Path) -> list[GoModule]:
"""
Vendor golang dependencies.

Expand Down Expand Up @@ -616,7 +614,7 @@ def parse_module_line(line: str) -> GoModule:


@tracer.start_as_current_span("_vendor_changed")
def _vendor_changed(git_dir: str, app_dir: str) -> bool:
def _vendor_changed(git_dir: Path, app_dir: str) -> bool:
"""Check for changes in the vendor directory."""
vendor = Path(app_dir).relative_to(git_dir).joinpath("vendor")
modules_txt = vendor / "modules.txt"
Expand All @@ -643,93 +641,31 @@ def _vendor_changed(git_dir: str, app_dir: str) -> bool:
return False


def _get_allowed_local_deps(module_name: str) -> List[str]:
"""
Get allowed local dependencies for module.

If module name contains a version and is not present in the allowlist, also try matching
without the version. E.g. if example.org/module/v2 is not present in the allowlist, return
allowed deps for example.org/module.
"""
allowlist = get_worker_config().cachito_gomod_file_deps_allowlist
allowed_deps = allowlist.get(module_name)
if allowed_deps is None:
versionless_module_name = MODULE_VERSION_RE.sub("", module_name)
allowed_deps = allowlist.get(versionless_module_name)
return allowed_deps or []


def _vet_local_deps(
def _vet_local_file_dep_paths(
dependencies: List[dict],
module_name: str,
allowed_patterns: List[str],
app_source_path: Path,
git_dir_path: str,
git_dir_path: Path,
) -> None:
"""
Fail if any dependency is replaced by a local path unless the module is allowlisted.

Also fail if the module is allowlisted but the path is absolute or outside repository.
"""
"""Fail if any local file dependency path is either outside the repository or absolute."""
for dep in dependencies:
name = dep["name"]
version = dep["version"]

if not version:
continue # go stdlib
if not version: # go stdlib
continue

if version.startswith("."):
log.debug(
"Module %s wants to replace %s with a local dependency: %s",
module_name,
name,
version,
)
_fail_unless_allowed(module_name, name, allowed_patterns)
_validate_local_dependency_path(app_source_path, git_dir_path, version)
resolved_dep_path = Path(app_source_path, version).resolve()
if not resolved_dep_path.is_relative_to(git_dir_path.resolve()):
raise ValidationError(
f"The local file dependency path {version} is outside the repository"
)
elif version.startswith("/") or PureWindowsPath(version).root:
# This will disallow paths starting with '/', '\' or '<drive letter>:\'
raise UnsupportedFeature(
f"Absolute paths to gomod dependencies are not supported: {version}"
)


def _validate_local_dependency_path(
app_source_path: Path, git_dir_path: str, dep_path: str
) -> None:
"""
Validate that the local dependency path is not outside the repository.

:param Path app_source_path: the full path to the application source code
:param str git_dir_path: the full path to the git repository
:param str dep_path: the relative path for local replacements (the dep version)
:raise ValidationError: if the local dependency path is invalid
"""
try:
resolved_dep_path = Path(app_source_path, dep_path).resolve()
resolved_dep_path.relative_to(Path(git_dir_path).resolve())
except ValueError:
raise ValidationError(f"The local dependency path {dep_path} is outside the repository")


def _fail_unless_allowed(module_name: str, package_name: str, allowed_patterns: List[str]):
"""
Fail unless the module is allowed to replace the package with a local dependency.

When packages are allowed to be replaced:
* package_name is a submodule of module_name
* package_name replacement is allowed according to allowed_patterns
"""
versionless_module_name = MODULE_VERSION_RE.sub("", module_name)
is_submodule = contains_package(versionless_module_name, package_name)
if not is_submodule and not any(fnmatch.fnmatch(package_name, pat) for pat in allowed_patterns):
raise UnsupportedFeature(
f"The module {module_name} is not allowed to replace {package_name} with a local "
f"dependency. Please contact the maintainers of this Cachito instance about adding "
"an exception."
)


def _set_full_local_dep_relpaths(pkg_deps: List[dict], main_module_deps: List[dict]):
"""
Set full relative paths for all local go-package dependencies.
Expand Down
Loading