From d1c29c8bb78e0ee5e00cad04d804416604b5c47d Mon Sep 17 00:00:00 2001
From: grossmj <grossmj@gns3.net>
Date: Sat, 18 Jul 2020 21:03:55 +0930
Subject: [PATCH] Resource constraints for Docker VMs.
---
gns3server/compute/docker/docker_vm.py | 28 +++-
.../handlers/api/compute/docker_handler.py | 33 ++--
gns3server/schemas/docker.py | 16 ++
gns3server/schemas/docker_template.py | 10 ++
tests/compute/docker/test_docker_vm.py | 144 ++++++++++++++++--
5 files changed, 200 insertions(+), 31 deletions(-)
diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py
index 940aeab6..6b3a644e 100644
--- a/gns3server/compute/docker/docker_vm.py
+++ b/gns3server/compute/docker/docker_vm.py
@@ -71,7 +71,7 @@ class DockerVM(BaseNode):
def __init__(self, name, node_id, project, manager, image, console=None, aux=None, start_command=None,
adapters=None, environment=None, console_type="telnet", console_resolution="1024x768",
- console_http_port=80, console_http_path="/", extra_hosts=None, extra_volumes=[]):
+ console_http_port=80, console_http_path="/", extra_hosts=None, extra_volumes=[], memory=0, cpus=0):
super().__init__(name, node_id, project, manager, console=console, aux=aux, allocate_aux=True, console_type=console_type)
@@ -94,6 +94,8 @@ class DockerVM(BaseNode):
self._console_websocket = None
self._extra_hosts = extra_hosts
self._extra_volumes = extra_volumes or []
+ self._memory = memory
+ self._cpus = cpus
self._permissions_fixed = False
self._display = None
self._closing = False
@@ -132,6 +134,8 @@ class DockerVM(BaseNode):
"node_directory": self.working_path,
"extra_hosts": self.extra_hosts,
"extra_volumes": self.extra_volumes,
+ "memory": self.memory,
+ "cpus": self.cpus
}
def _get_free_display_port(self):
@@ -211,6 +215,22 @@ class DockerVM(BaseNode):
def extra_volumes(self, extra_volumes):
self._extra_volumes = extra_volumes
+ @property
+ def memory(self):
+ return self._memory
+
+ @memory.setter
+ def memory(self, memory):
+ self._memory = memory
+
+ @property
+ def cpus(self):
+ return self._cpus
+
+ @cpus.setter
+ def cpus(self, cpus):
+ self._cpus = cpus
+
async def _get_container_state(self):
"""
Returns the container state (e.g. running, paused etc.)
@@ -328,6 +348,10 @@ class DockerVM(BaseNode):
if image_infos is None:
raise DockerError("Cannot get information for image '{}', please try again.".format(self._image))
+ available_cpus = psutil.cpu_count(logical=True)
+ if self._cpus > available_cpus:
+ raise DockerError("You have allocated too many CPUs for the Docker container (max available is {} CPUs)".format(available_cpus))
+
params = {
"Hostname": self._name,
"Name": self._name,
@@ -340,6 +364,8 @@ class DockerVM(BaseNode):
"CapAdd": ["ALL"],
"Privileged": True,
"Binds": self._mount_binds(image_infos),
+ "Memory": self._memory * (1024 * 1024), # convert memory to bytes
+ "NanoCpus": int(self._cpus * 1e9) # convert cpus to nano cpus
},
"Volumes": {},
"Env": ["container=docker"], # Systemd compliant: https://github.com/GNS3/gns3-server/issues/573
diff --git a/gns3server/handlers/api/compute/docker_handler.py b/gns3server/handlers/api/compute/docker_handler.py
index e510a9f2..b7935071 100644
--- a/gns3server/handlers/api/compute/docker_handler.py
+++ b/gns3server/handlers/api/compute/docker_handler.py
@@ -49,20 +49,22 @@ class DockerHandler:
async def create(request, response):
docker_manager = Docker.instance()
container = await docker_manager.create_node(request.json.pop("name"),
- request.match_info["project_id"],
- request.json.get("node_id"),
- image=request.json.pop("image"),
- start_command=request.json.get("start_command"),
- environment=request.json.get("environment"),
- adapters=request.json.get("adapters"),
- console=request.json.get("console"),
- console_type=request.json.get("console_type"),
- console_resolution=request.json.get("console_resolution", "1024x768"),
- console_http_port=request.json.get("console_http_port", 80),
- console_http_path=request.json.get("console_http_path", "/"),
- aux=request.json.get("aux"),
- extra_hosts=request.json.get("extra_hosts"),
- extra_volumes=request.json.get("extra_volumes"))
+ request.match_info["project_id"],
+ request.json.get("node_id"),
+ image=request.json.pop("image"),
+ start_command=request.json.get("start_command"),
+ environment=request.json.get("environment"),
+ adapters=request.json.get("adapters"),
+ console=request.json.get("console"),
+ console_type=request.json.get("console_type"),
+ console_resolution=request.json.get("console_resolution", "1024x768"),
+ console_http_port=request.json.get("console_http_port", 80),
+ console_http_path=request.json.get("console_http_path", "/"),
+ aux=request.json.get("aux"),
+ extra_hosts=request.json.get("extra_hosts"),
+ extra_volumes=request.json.get("extra_volumes"),
+ memory=request.json.get("memory", 0),
+ cpus=request.json.get("cpus", 0))
for name, value in request.json.items():
if name != "node_id":
if hasattr(container, name) and getattr(container, name) != value:
@@ -317,7 +319,8 @@ class DockerHandler:
props = [
"name", "console", "aux", "console_type", "console_resolution",
"console_http_port", "console_http_path", "start_command",
- "environment", "adapters", "extra_hosts", "extra_volumes"
+ "environment", "adapters", "extra_hosts", "extra_volumes",
+ "memory", "cpus"
]
changed = False
diff --git a/gns3server/schemas/docker.py b/gns3server/schemas/docker.py
index 6cea166a..db56a61a 100644
--- a/gns3server/schemas/docker.py
+++ b/gns3server/schemas/docker.py
@@ -103,6 +103,14 @@ DOCKER_CREATE_SCHEMA = {
"type": "string"
}
},
+ "memory": {
+ "description": "Maximum amount of memory the container can use in MB",
+ "type": "integer",
+ },
+ "cpus": {
+ "description": "Maximum amount of CPU resources the container can use",
+ "type": "number",
+ },
"container_id": {
"description": "Docker container ID Read only",
"type": "string",
@@ -214,6 +222,14 @@ DOCKER_OBJECT_SCHEMA = {
"type": "string",
}
},
+ "memory": {
+ "description": "Maximum amount of memory the container can use in MB",
+ "type": "integer",
+ },
+ "cpus": {
+ "description": "Maximum amount of CPU resources the container can use",
+ "type": "number",
+ },
"node_directory": {
"description": "Path to the node working directory Read only",
"type": "string"
diff --git a/gns3server/schemas/docker_template.py b/gns3server/schemas/docker_template.py
index dd569bcb..40735cb2 100644
--- a/gns3server/schemas/docker_template.py
+++ b/gns3server/schemas/docker_template.py
@@ -82,6 +82,16 @@ DOCKER_TEMPLATE_PROPERTIES = {
"type": "array",
"default": []
},
+ "memory": {
+ "description": "Maximum amount of memory the container can use in MB",
+ "type": "integer",
+ "default": 0
+ },
+ "cpus": {
+ "description": "Maximum amount of CPU resources the container can use",
+ "type": "number",
+ "default": 0
+ },
"custom_adapters": CUSTOM_ADAPTERS_ARRAY_SCHEMA
}
diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py
index 1cefc7c2..1f639fb3 100644
--- a/tests/compute/docker/test_docker_vm.py
+++ b/tests/compute/docker/test_docker_vm.py
@@ -66,6 +66,8 @@ def test_json(vm, compute_project):
'console_http_path': '/',
'extra_hosts': None,
'extra_volumes': [],
+ 'memory': 0,
+ 'cpus': 0,
'aux': vm.aux,
'start_command': vm.start_command,
'environment': vm.environment,
@@ -104,7 +106,9 @@ async def test_create(compute_project, manager):
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -143,7 +147,9 @@ async def test_create_with_tag(compute_project, manager):
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -186,7 +192,9 @@ async def test_create_vnc(compute_project, manager):
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
'/tmp/.X11-unix/:/tmp/.X11-unix/'
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -301,7 +309,9 @@ async def test_create_start_cmd(compute_project, manager):
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"Entrypoint": ["/gns3/init.sh"],
@@ -400,7 +410,9 @@ async def test_create_image_not_available(compute_project, manager):
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -444,7 +456,9 @@ async def test_create_with_user(compute_project, manager):
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -528,7 +542,9 @@ async def test_create_with_extra_volumes_duplicate_1_image(compute_project, mana
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -568,7 +584,9 @@ async def test_create_with_extra_volumes_duplicate_2_user(compute_project, manag
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -608,7 +626,9 @@ async def test_create_with_extra_volumes_duplicate_3_subdir(compute_project, man
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -648,7 +668,9 @@ async def test_create_with_extra_volumes_duplicate_4_backslash(compute_project,
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
"{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -687,7 +709,9 @@ async def test_create_with_extra_volumes_duplicate_5_subdir_issue_1595(compute_p
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -726,7 +750,9 @@ async def test_create_with_extra_volumes_duplicate_6_subdir_issue_1595(compute_p
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -773,7 +799,9 @@ async def test_create_with_extra_volumes(compute_project, manager):
"{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
"{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")),
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -996,7 +1024,9 @@ async def test_update(vm):
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -1065,7 +1095,9 @@ async def test_update_running(vm):
"{}:/gns3:ro".format(get_resource("compute/docker/resources")),
"{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
],
- "Privileged": True
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 0
},
"Volumes": {},
"NetworkDisabled": True,
@@ -1422,3 +1454,85 @@ async def test_read_console_output_with_binary_mode(vm):
with asyncio_patch('gns3server.compute.docker.docker_vm.DockerVM.stop'):
await vm._read_console_output(input_stream, output_stream)
output_stream.feed_data.assert_called_once_with(b"test")
+
+
+async def test_cpus(compute_project, manager):
+
+ response = {
+ "Id": "e90e34656806",
+ "Warnings": []
+ }
+ with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]):
+ with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+ vm = DockerVM("test", str(uuid.uuid4()), compute_project, manager, "ubuntu:latest", cpus=0.5)
+ await vm.create()
+ mock.assert_called_with("POST", "containers/create", data={
+ "Tty": True,
+ "OpenStdin": True,
+ "StdinOnce": False,
+ "HostConfig":
+ {
+ "CapAdd": ["ALL"],
+ "Binds": [
+ "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+ "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
+ ],
+ "Privileged": True,
+ "Memory": 0,
+ "NanoCpus": 500000000
+ },
+ "Volumes": {},
+ "NetworkDisabled": True,
+ "Name": "test",
+ "Hostname": "test",
+ "Image": "ubuntu:latest",
+ "Env": [
+ "container=docker",
+ "GNS3_MAX_ETHERNET=eth0",
+ "GNS3_VOLUMES=/etc/network"
+ ],
+ "Entrypoint": ["/gns3/init.sh"],
+ "Cmd": ["/bin/sh"]
+ })
+ assert vm._cid == "e90e34656806"
+
+
+async def test_memory(compute_project, manager):
+
+ response = {
+ "Id": "e90e34656806",
+ "Warnings": []
+ }
+ with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]):
+ with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+ vm = DockerVM("test", str(uuid.uuid4()), compute_project, manager, "ubuntu:latest", memory=32)
+ await vm.create()
+ mock.assert_called_with("POST", "containers/create", data={
+ "Tty": True,
+ "OpenStdin": True,
+ "StdinOnce": False,
+ "HostConfig":
+ {
+ "CapAdd": ["ALL"],
+ "Binds": [
+ "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+ "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
+ ],
+ "Privileged": True,
+ "Memory": 33554432, # 32MB in bytes
+ "NanoCpus": 0
+ },
+ "Volumes": {},
+ "NetworkDisabled": True,
+ "Name": "test",
+ "Hostname": "test",
+ "Image": "ubuntu:latest",
+ "Env": [
+ "container=docker",
+ "GNS3_MAX_ETHERNET=eth0",
+ "GNS3_VOLUMES=/etc/network"
+ ],
+ "Entrypoint": ["/gns3/init.sh"],
+ "Cmd": ["/bin/sh"]
+ })
+ assert vm._cid == "e90e34656806"