From 69667521e2c369d27d84da86d41a7719dfe25ece Mon Sep 17 00:00:00 2001 From: Alessandro Pirastru <57262788+Bloodis94@users.noreply.github.com> Date: Sun, 27 Apr 2025 16:01:29 +0200 Subject: [PATCH] fix(install/gpu):Fix docker not being able to leverage the GPU on systems that have SELinux Enforced (#5252) * Update installation script for improved compatibility and clarity - Renamed VERSION to LOCALAI_VERSION to avoid conflicts with system variables. - Enhanced NVIDIA and CUDA repository installation for DNF5 compatibility. - Adjusted default Fedora version handling for CUDA installation. - Updated Docker image tag handling to use LOCALAI_VERSION consistently. - Improved logging messages for repository and LocalAI binary downloads. - Added a temporary bypass for nvidia-smi installation on Fedora Cloud Edition. * feat: Add SELinux configuration for NVIDIA GPU support in containers - Introduced `enable_selinux_container_booleans` function to handle SELinux configuration changes for GPU access. - Included user confirmation prompt to enable SELinux `container_use_devices` boolean due to security implications. - Added NVIDIA Container Runtime to Docker runtimes and restarted Docker to ensure proper GPU support. - Applied SELinux adjustments conditionally for Fedora, RHEL, CentOS, Rocky, and openSUSE distributions. Signed-off-by: Alessandro Pirastru * fix: Correct SELinux boolean parsing and add loop break - Fixed incorrect parsing of `container_use_devices` boolean by changing the awk field from `$2` to `$3` to retrieve the correct value. - Added a `break` statement after enabling the SELinux boolean to prevent unnecessary loop iterations after user prompt. Signed-off-by: Alessandro Pirastru * fix: typo in install.sh Signed-off-by: Alessandro Pirastru <57262788+Bloodis94@users.noreply.github.com> --------- Signed-off-by: Alessandro Pirastru Signed-off-by: Alessandro Pirastru <57262788+Bloodis94@users.noreply.github.com> --- docs/static/install.sh | 69 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/docs/static/install.sh b/docs/static/install.sh index b36b0781..c39c6ba3 100755 --- a/docs/static/install.sh +++ b/docs/static/install.sh @@ -307,9 +307,53 @@ install_container_toolkit_yum() { $SUDO $PACKAGE_MANAGER install -y nvidia-container-toolkit } +# Fedora, Rhel and other distro ships tunable SELinux booleans in the container-selinux policy to control device access. +# In particular, enabling container_use_devices allows containers to use arbitrary host device labels (including GPU devices) +# ref: https://github.com/containers/ramalama/blob/main/docs/ramalama-cuda.7.md#expected-output +enable_selinux_container_booleans() { + + # Check SELinux mode + SELINUX_MODE=$(getenforce) + + if [ "$SELINUX_MODE" == "Enforcing" ]; then + # Check the status of container_use_devices + CONTAINER_USE_DEVICES=$(getsebool container_use_devices | awk '{print $3}') + + if [ "$CONTAINER_USE_DEVICES" == "off" ]; then + + #We want to give the user the choice to enable the SE booleans since it is a security config + warn "+-----------------------------------------------------------------------------------------------------------+" + warn "| WARNING: |" + warn "| Your distribution ships tunable SELinux booleans in the container-selinux policy to control device access.|" + warn "| In particular, enabling \"container_use_devices\" allows containers to use arbitrary host device labels |" + warn "| (including GPU devices). |" + warn "| This script can try to enable them enabling the \"container_use_devices\" flag. |" + warn "| |" + warn "| Otherwise you can exit the install script and enable them yourself. |" + warn "+-----------------------------------------------------------------------------------------------------------+" + + while true; do + choice_warn "I understand that this script is going to change my SELinux configs, which is a security risk: (yes/exit) "; + read Answer + + if [ "$Answer" = "yes" ]; then + warn "Enabling \"container_use_devices\" persistently..." + $SUDO setsebool -P container_use_devices 1 + + break + elif [ "$Answer" = "exit" ]; then + aborted + else + warn "Invalid choice. Please enter 'yes' or 'exit'." + fi + done + fi + fi +} + # ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#installing-with-apt install_container_toolkit_apt() { - info 'Installing NVIDIA container toolkit repository...' + info 'Installing NVIDIA container toolkit repository...' curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | $SUDO gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ @@ -350,6 +394,29 @@ install_container_toolkit() { opensuse*|suse*) install_container_toolkit_zypper ;; *) echo "Could not install nvidia container toolkit - unknown OS" ;; esac + + # after installing the toolkit we need to add it to the docker runtimes, otherwise even with --gpu all + # the container would still run with runc and would not have access to nvidia-smi + # ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#configuring-docker + info "Adding NVIDIA Container Runtime to Docker runtimes..." + $SUDO nvidia-ctk runtime configure --runtime=docker + + info "Restarting Docker Daemon" + $SUDO systemctl restart docker + + # The NVML error arises because SELinux blocked the container’s attempts to open the GPU devices or related libraries. + # Without relaxing SELinux for the container, GPU commands like nvidia-smi report “Insufficient Permissions” + # This has been noted in NVIDIA’s documentation: + # ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.13.5/install-guide.html#id2 + # ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/troubleshooting.html#nvml-insufficient-permissions-and-selinux + case $OS_NAME in + fedora|rhel|centos|rocky) + enable_selinux_container_booleans + ;; + opensuse-tumbleweed) + enable_selinux_container_booleans + ;; + esac } # ref: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#rhel-7-centos-7