Add core architecture patterns and GIS/weather components from AIOS-Public

2025-10-16 13:14:30 -05:00
parent 782eec63a5
commit 5887f4e729
32 changed files with 1970 additions and 1 deletions
--- a/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/Dockerfile
+++ b/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/Dockerfile
@@ -0,0 +1,72 @@
+FROM debian:bookworm-slim
+
+# Avoid prompts from apt
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install base packages for weather tools
+RUN apt-get update && apt-get install -y \
+    bash \
+    curl \
+    wget \
+    git \
+    python3 \
+    python3-pip \
+    build-essential \
+    sudo \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create symbolic link for python
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+# Install weather data processing tools
+RUN apt-get update && apt-get install -y \
+    ftp \
+    gfortran \
+    libgfortran5 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python weather libraries
+RUN pip3 install --break-system-packages \
+    xarray \
+    cfgrib \
+    netcdf4 \
+    metpy \
+    siphon \
+    numpy \
+    pandas \
+    scipy \
+    matplotlib \
+    seaborn \
+    requests \
+    cdo-python
+
+# Install Climate Data Operators (CDO)
+RUN apt-get update && apt-get install -y \
+    cdo \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install additional tools for weather data operations
+RUN apt-get update && apt-get install -y \
+    nco \
+    ncl-ncarg \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install R for statistical analysis
+RUN apt-get update && apt-get install -y \
+    r-base \
+    r-base-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install R weather packages
+RUN R --slave -e "install.packages(c('raster', 'rgdal', 'ncdf4', 'rasterVis', 'ncmeta'), repos='https://cran.rstudio.com/', dependencies=TRUE)"
+
+# Add entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Create a working directory
+WORKDIR /workspace
+
+# Use the entrypoint script to handle user creation
+ENTRYPOINT ["/entrypoint.sh"]
--- a/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/README.md
+++ b/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/README.md
@@ -0,0 +1,121 @@
+# TSYS-AIOS-GIS-Tools-Weather-Base Container
+
+This container is part of the TSYS-AIOS-GIS project and provides a base weather data processing environment.
+
+## Overview
+
+The TSYS-AIOS-GIS-Tools-Weather-Base container is designed for weather data processing and analysis tasks. It includes essential tools for handling weather data formats (GRIB, NetCDF), accessing weather APIs, and performing meteorological calculations.
+
+## Tools Included
+
+### Core Tools
+- **Base OS**: Debian Bookworm slim
+- **Shell**: Bash
+- **Programming Languages**:
+  - Python 3 with meteorological libraries
+  - R with weather analysis packages
+
+### Weather Data Processing Libraries
+- **xarray**: Multi-dimensional data in Python
+- **cfgrib**: GRIB format handling
+- **netCDF4**: NetCDF file handling
+- **MetPy**: Meteorological calculations
+- **Siphon**: Access to weather data from various sources
+- **Numpy/Pandas/Scipy**: Scientific computing libraries
+
+### Climate Data Operators
+- **CDO (Climate Data Operators)**: Tools for climate data processing
+- **NCO (NetCDF Operators)**: Operators for NetCDF files
+- **NCL (NCAR Command Language)**: For complex climate analysis
+
+### Visualization & Analysis
+- **Matplotlib/Seaborn**: Statistical plots
+- **Requests**: HTTP library for API access
+- **R packages**: raster, rgdal, ncdf4, rasterVis, ncmeta
+
+### Additional Tools
+- **FTP client**: For bulk weather data downloads
+- **GFortran**: For compiling Fortran-based weather tools
+
+## Usage
+
+### Building the Weather Base Container
+```bash
+# From this directory
+cd /home/localuser/AIWorkspace/TSYS-AIOS-GIS/Docker/TSYS-AIOS-GIS-Tools-Weather-Base
+
+# Use the wrapper script to automatically detect and set user IDs
+./docker-compose-wrapper.sh build
+
+# Or run commands in the weather container with automatic user mapping
+./docker-compose-wrapper.sh run tsys-weather-base [command]
+
+# Example: Process a GRIB file with cfgrib
+./docker-compose-wrapper.sh run tsys-weather-base python3 -c "import xarray as xr; ds = xr.open_dataset('file.grib', engine='cfgrib'); print(ds)"
+
+# Example: Use CDO to process NetCDF files
+./docker-compose-wrapper.sh run tsys-weather-base cdo info /workspace/weather_data.nc
+
+# Example: Start Python with weather libraries
+./docker-compose-wrapper.sh run tsys-weather-base python3
+```
+
+### Using with docker-compose directly
+```bash
+# Set environment variables and run docker-compose directly
+LOCAL_USER_ID=$(id -u) LOCAL_GROUP_ID=$(id -g) docker-compose up --build
+
+# Or export variables first
+export LOCAL_USER_ID=$(id -u)
+export LOCAL_GROUP_ID=$(id -g) 
+docker-compose up
+```
+
+### Using the wrapper script
+```bash
+# Build and start the base weather container with automatic user mapping
+./docker-compose-wrapper.sh up --build
+
+# Start without rebuilding
+./docker-compose-wrapper.sh up
+
+# View container status
+./docker-compose-wrapper.sh ps
+
+# Stop containers
+./docker-compose-wrapper.sh down
+```
+
+## Weather Data Processing Workflows
+
+This container is designed to handle:
+- GRIB data format processing
+- NetCDF data analysis
+- NOAA and European weather API integration
+- Bulk data download via HTTP/FTP
+- Meteorological calculations
+- Climate data processing with CDO/NCO
+
+## User ID Mapping (For File Permissions)
+
+The container automatically detects and uses the host user's UID and GID to ensure proper file permissions. This means:
+
+- Files created inside the container will have the correct ownership on the host
+- No more root-owned files after container operations
+- Works across different environments (development, production servers)
+
+The container detects the user ID from the mounted workspace volume. If needed, you can override the default values by setting environment variables:
+
+```bash
+# Set specific user ID and group ID before running docker-compose
+export LOCAL_USER_ID=1000
+export LOCAL_GROUP_ID=1000
+docker-compose up
+```
+
+Or run with inline environment variables:
+```bash
+LOCAL_USER_ID=1000 LOCAL_GROUP_ID=1000 docker-compose up
+```
+
+The container runs as a non-root user named `TSYS-Tools` with the detected host user's UID/GID.
--- a/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/docker-compose-wrapper.sh
+++ b/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/docker-compose-wrapper.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+# docker-compose-wrapper.sh - Wrapper script to detect host UID/GID and run docker-compose
+
+set -e  # Exit on any error
+
+# Detect the UID and GID of the user that owns the workspace directory (parent directory)
+WORKSPACE_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
+echo "Detecting user ID from workspace directory: $WORKSPACE_DIR"
+
+if [ -d "$WORKSPACE_DIR" ]; then
+    DETECTED_USER_ID=$(stat -c %u "$WORKSPACE_DIR" 2>/dev/null || echo 0)
+    DETECTED_GROUP_ID=$(stat -c %g "$WORKSPACE_DIR" 2>/dev/null || echo 0)
+    
+    # If detection failed, try current user
+    if [ "$DETECTED_USER_ID" = "0" ]; then
+        DETECTED_USER_ID=$(id -u)
+        DETECTED_GROUP_ID=$(id -g)
+    fi
+else
+    # Fallback to current user if workspace directory doesn't exist
+    DETECTED_USER_ID=$(id -u)
+    DETECTED_GROUP_ID=$(id -g)
+fi
+
+echo "Detected USER_ID=$DETECTED_USER_ID and GROUP_ID=$DETECTED_GROUP_ID"
+
+# Set environment variables for docker-compose
+export LOCAL_USER_ID=$DETECTED_USER_ID
+export LOCAL_GROUP_ID=$DETECTED_GROUP_ID
+
+# Show usage information
+echo ""
+echo "Usage: $0 [build|up|run <service> <command>|exec <service> <command>|down|ps]"
+echo ""
+echo "Examples:"
+echo "  $0 up                    # Start services"
+echo "  $0 build                 # Build containers"
+echo "  $0 run tsys-weather-base bash  # Run command in container"
+echo "  $0 down                  # Stop and remove containers"
+echo ""
+
+# Check if docker compose (new format) or docker-compose (old format) is available
+if command -v docker &> /dev/null && docker compose version &> /dev/null; then
+    # Use new docker compose format
+    if [ $# -eq 0 ]; then
+        echo "No command provided. Running 'docker compose up'..."
+        docker compose up
+    else
+        # Execute the provided docker compose command
+        echo "Running: docker compose $*"
+        docker compose "$@"
+    fi
+elif command -v docker-compose &> /dev/null; then
+    # Fallback to old docker-compose format
+    if [ $# -eq 0 ]; then
+        echo "No command provided. Running 'docker-compose up'..."
+        docker-compose up
+    else
+        # Execute the provided docker-compose command
+        echo "Running: docker-compose $*"
+        docker-compose "$@"
+    fi
+else
+    echo "Error: Neither 'docker compose' nor 'docker-compose' command found."
+    echo "Please install Docker Compose to use this script."
+    exit 1
+fi
--- a/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/docker-compose.yml
+++ b/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/docker-compose.yml
@@ -0,0 +1,18 @@
+version: '3.8'
+
+services:
+  tsys-weather-base:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: TSYS-AIOS-GIS-Tools-Weather-Base
+    image: tsys-aios-gis-tools-weather-base:latest
+    volumes:
+      - ../../../:/workspace:rw
+    working_dir: /workspace
+    stdin_open: true
+    tty: true
+    environment:
+      - LOCAL_USER_ID=${LOCAL_USER_ID:-1000}
+      - LOCAL_GROUP_ID=${LOCAL_GROUP_ID:-1000}
+    user: "${LOCAL_USER_ID:-1000}:${LOCAL_GROUP_ID:-1000}"
--- a/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/entrypoint.sh
+++ b/Docker/TSYS-AIOS-GIS-Tools-Weather-Base/entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# entrypoint.sh - Entrypoint script to handle user creation and permission setup at runtime
+
+# Set default values if not provided
+USER_ID=${LOCAL_USER_ID:-$(id -u 1000)}
+GROUP_ID=${LOCAL_GROUP_ID:-$(id -g 1000)}
+
+# In case the environment variables are not set properly, detect them from the workspace volume
+if [ "$USER_ID" = "$(id -u 0)" ] || [ "$USER_ID" = "0" ]; then
+    # Detect the UID and GID of the user that owns the workspace directory
+    if [ -d "/workspace" ]; then
+        USER_ID=$(stat -c %u /workspace 2>/dev/null || echo 1000)
+        GROUP_ID=$(stat -c %g /workspace 2>/dev/null || echo 1000)
+    else
+        USER_ID=${LOCAL_USER_ID:-1000}
+        GROUP_ID=${LOCAL_GROUP_ID:-1000}
+    fi
+fi
+
+echo "Starting with USER_ID=$USER_ID and GROUP_ID=$GROUP_ID"
+
+# Create the group with specified GID
+groupadd -f -g $GROUP_ID -o TSYS-Tools 2>/dev/null || groupmod -g $GROUP_ID -o TSYS-Tools
+
+# Create the user with specified UID and add to the group
+useradd -f -u $USER_ID -g $GROUP_ID -m -s /bin/bash -o TSYS-Tools 2>/dev/null || usermod -u $USER_ID -g $GROUP_ID -o TSYS-Tools
+
+# Add user to sudo group for any necessary operations
+usermod -aG sudo TSYS-Tools 2>/dev/null || true
+
+# Make sure workspace directory exists and has proper permissions
+mkdir -p /workspace
+chown -R $USER_ID:$GROUP_ID /workspace
+
+# Set up proper permissions for .local (if they exist)
+mkdir -p /home/TSYS-Tools/.local
+chown $USER_ID:$GROUP_ID /home/TSYS-Tools/.local
+
+# Set up proper permissions for R (if they exist)
+mkdir -p /home/TSYS-Tools/R
+chown $USER_ID:$GROUP_ID /home/TSYS-Tools/R
+
+# If there are additional arguments, run them as the created user
+if [ $# -gt 0 ]; then
+    exec su -p TSYS-Tools -c "$*"
+else
+    # Otherwise start an interactive bash shell as the created user
+    exec su -p TSYS-Tools -c "/bin/bash"
+fi