Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Install Docker resources in writable location #2355

Merged
merged 1 commit into from
Feb 14, 2024
Merged

Install Docker resources in writable location #2355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions gns3server/compute/docker/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@
Docker server module.
"""

import os
import sys
import json
import asyncio
import logging
import aiohttp
import shutil
import platformdirs

from gns3server.utils import parse_version
from gns3server.utils.asyncio import locking
from gns3server.compute.base_manager import BaseManager
Expand Down Expand Up @@ -55,6 +59,62 @@ def __init__(self):
self._session = None
self._api_version = DOCKER_MINIMUM_API_VERSION

@staticmethod
async def install_busybox(dst_dir):

dst_busybox = os.path.join(dst_dir, "bin", "busybox")
if os.path.isfile(dst_busybox):
return
for busybox_exec in ("busybox-static", "busybox.static", "busybox"):
busybox_path = shutil.which(busybox_exec)
if busybox_path:
try:
# check that busybox is statically linked
# (dynamically linked busybox will fail to run in a container)
proc = await asyncio.create_subprocess_exec(
"ldd",
busybox_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL
)
stdout, _ = await proc.communicate()
if proc.returncode == 1:
# ldd returns 1 if the file is not a dynamic executable
log.info(f"Installing busybox from '{busybox_path}' to '{dst_busybox}'")
shutil.copy2(busybox_path, dst_busybox, follow_symlinks=True)
return
else:
log.warning(f"Busybox '{busybox_path}' is dynamically linked\n"
f"{stdout.decode('utf-8', errors='ignore').strip()}")
except OSError as e:
raise DockerError(f"Could not install busybox: {e}")
raise DockerError("No busybox executable could be found")

@staticmethod
def resources_path():
"""
Get the Docker resources storage directory
"""

appname = vendor = "GNS3"
docker_resources_dir = os.path.join(platformdirs.user_data_dir(appname, vendor, roaming=True), "docker", "resources")
os.makedirs(docker_resources_dir, exist_ok=True)
return docker_resources_dir

async def install_resources(self):
"""
Copy the necessary resources to a writable location and install busybox
"""

try:
dst_path = self.resources_path()
log.info(f"Installing Docker resources in '{dst_path}'")
from gns3server.controller import Controller
Controller.instance().install_resource_files(dst_path, "compute/docker/resources")
await self.install_busybox(dst_path)
except OSError as e:
raise DockerError(f"Could not install Docker resources to {dst_path}: {e}")

async def _check_connection(self):

if not self._connected:
Expand Down
13 changes: 9 additions & 4 deletions gns3server/compute/docker/docker_vm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,10 +242,13 @@ def _mount_binds(self, image_info):
:returns: Return the path that we need to map to local folders
"""

resources = get_resource("compute/docker/resources")
if not os.path.exists(resources):
raise DockerError("{} is missing can't start Docker containers".format(resources))
binds = ["{}:/gns3:ro".format(resources)]
try:
resources_path = self.manager.resources_path()
except OSError as e:
raise DockerError(f"Cannot access resources: {e}")

log.info(f'Mount resources from "{resources_path}"')
binds = ["{}:/gns3:ro".format(resources_path)]

# We mount our own etc/network
try:
Expand Down Expand Up @@ -460,6 +463,8 @@ async def start(self):
Starts this Docker container.
"""

await self.manager.install_resources()

try:
state = await self._get_container_state()
except DockerHttp404Error:
Expand Down
9 changes: 6 additions & 3 deletions gns3server/controller/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,9 +297,12 @@ def install_resource_files(dst_path, resource_name):
else:
for entry in importlib_resources.files('gns3server').joinpath(resource_name).iterdir():
full_path = os.path.join(dst_path, entry.name)
if entry.is_file() and not os.path.exists(full_path):
log.debug(f'Installing {resource_name} resource file "{entry.name}" to "{full_path}"')
shutil.copy(str(entry), os.path.join(dst_path, entry.name))
if not os.path.exists(full_path):
if entry.is_file():
log.debug(f'Installing {resource_name} resource file "{entry.name}" to "{full_path}"')
shutil.copy(str(entry), os.path.join(dst_path, entry.name))
elif entry.is_dir():
os.makedirs(full_path, exist_ok=True)

def _install_base_configs(self):
"""
Expand Down
22 changes: 0 additions & 22 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,28 +43,6 @@ def run_tests(self):
sys.exit(errcode)


BUSYBOX_PATH = "gns3server/compute/docker/resources/bin/busybox"


def copy_busybox():
if not sys.platform.startswith("linux"):
return
if os.path.isfile(BUSYBOX_PATH):
return
for bb_cmd in ("busybox-static", "busybox.static", "busybox"):
bb_path = shutil.which(bb_cmd)
if bb_path:
if subprocess.call(["ldd", bb_path],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL):
shutil.copy2(bb_path, BUSYBOX_PATH, follow_symlinks=True)
break
else:
raise SystemExit("No static busybox found")


copy_busybox()
dependencies = open("requirements.txt", "r").read().splitlines()

setup(
Expand Down
50 changes: 50 additions & 0 deletions tests/compute/docker/test_docker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import asyncio
import pytest
from unittest.mock import MagicMock, patch

Expand Down Expand Up @@ -200,3 +201,52 @@ async def test_docker_check_connection_docker_preferred_version_against_older(vm
vm._connected = False
await vm._check_connection()
assert vm._api_version == DOCKER_MINIMUM_API_VERSION


@pytest.mark.asyncio
async def test_install_busybox():

mock_process = MagicMock()
mock_process.returncode = 1 # means that busybox is not dynamically linked
mock_process.communicate = AsyncioMagicMock(return_value=(b"", b"not a dynamic executable"))

with patch("gns3server.compute.docker.os.path.isfile", return_value=False):
with patch("gns3server.compute.docker.shutil.which", return_value="/usr/bin/busybox"):
with asyncio_patch("gns3server.compute.docker.asyncio.create_subprocess_exec", return_value=mock_process) as create_subprocess_mock:
with patch("gns3server.compute.docker.shutil.copy2") as copy2_mock:
dst_dir = Docker.resources_path()
await Docker.install_busybox(dst_dir)
create_subprocess_mock.assert_called_with(
"ldd",
"/usr/bin/busybox",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
assert copy2_mock.called


@pytest.mark.asyncio
async def test_install_busybox_dynamic_linked():

mock_process = MagicMock()
mock_process.returncode = 0 # means that busybox is dynamically linked
mock_process.communicate = AsyncioMagicMock(return_value=(b"Dynamically linked library", b""))

with patch("os.path.isfile", return_value=False):
with patch("gns3server.compute.docker.shutil.which", return_value="/usr/bin/busybox"):
with asyncio_patch("gns3server.compute.docker.asyncio.create_subprocess_exec", return_value=mock_process):
with pytest.raises(DockerError) as e:
dst_dir = Docker.resources_path()
await Docker.install_busybox(dst_dir)
assert str(e.value) == "No busybox executable could be found"


@pytest.mark.asyncio
async def test_install_busybox_no_executables():

with patch("gns3server.compute.docker.os.path.isfile", return_value=False):
with patch("gns3server.compute.docker.shutil.which", return_value=None):
with pytest.raises(DockerError) as e:
dst_dir = Docker.resources_path()
await Docker.install_busybox(dst_dir)
assert str(e.value) == "No busybox executable could be found"
36 changes: 17 additions & 19 deletions tests/compute/docker/test_docker_vm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@

from gns3server.ubridge.ubridge_error import UbridgeNamespaceError
from gns3server.compute.docker.docker_vm import DockerVM
from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error, DockerHttp304Error
from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error
from gns3server.compute.docker import Docker
from gns3server.utils.get_resource import get_resource


from unittest.mock import patch, MagicMock, call

Expand Down Expand Up @@ -101,7 +99,7 @@ async def test_create(compute_project, manager):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
"Privileged": True
Expand Down Expand Up @@ -139,7 +137,7 @@ async def test_create_with_tag(compute_project, manager):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
"Privileged": True
Expand Down Expand Up @@ -180,7 +178,7 @@ async def test_create_vnc(compute_project, manager):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"/tmp/.X11-unix/X{0}:/tmp/.X11-unix/X{0}:ro".format(vm._display)
],
Expand Down Expand Up @@ -310,7 +308,7 @@ async def test_create_start_cmd(compute_project, manager):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
"Privileged": True
Expand Down Expand Up @@ -408,7 +406,7 @@ async def information():
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
"Privileged": True
Expand Down Expand Up @@ -451,7 +449,7 @@ async def test_create_with_user(compute_project, manager):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
"Privileged": True
Expand Down Expand Up @@ -533,7 +531,7 @@ async def test_create_with_extra_volumes_duplicate_1_image(compute_project, mana
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
],
Expand Down Expand Up @@ -572,7 +570,7 @@ async def test_create_with_extra_volumes_duplicate_2_user(compute_project, manag
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
],
Expand Down Expand Up @@ -611,7 +609,7 @@ async def test_create_with_extra_volumes_duplicate_3_subdir(compute_project, man
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
],
Expand Down Expand Up @@ -650,7 +648,7 @@ async def test_create_with_extra_volumes_duplicate_4_backslash(compute_project,
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
],
Expand Down Expand Up @@ -689,7 +687,7 @@ async def test_create_with_extra_volumes_duplicate_5_subdir_issue_1595(compute_p
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
],
"Privileged": True
Expand Down Expand Up @@ -727,7 +725,7 @@ async def test_create_with_extra_volumes_duplicate_6_subdir_issue_1595(compute_p
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
],
"Privileged": True
Expand Down Expand Up @@ -771,7 +769,7 @@ async def test_create_with_extra_volumes(compute_project, manager):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
"{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")),
Expand Down Expand Up @@ -996,7 +994,7 @@ async def test_update(vm):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
"Privileged": True
Expand Down Expand Up @@ -1064,7 +1062,7 @@ async def test_update_running(vm):
{
"CapAdd": ["ALL"],
"Binds": [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
"Privileged": True
Expand Down Expand Up @@ -1325,7 +1323,7 @@ async def test_mount_binds(vm):

dst = os.path.join(vm.working_dir, "test/experimental")
assert vm._mount_binds(image_infos) == [
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3:ro".format(Docker.resources_path()),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes{}".format(dst, "/test/experimental")
]
Expand Down
Loading