Add core architecture patterns and GIS/weather components from AIOS-Public

2025-10-16 13:14:30 -05:00
parent 782eec63a5
commit 5887f4e729
32 changed files with 1970 additions and 1 deletions
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/Dockerfile
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/Dockerfile
@@ -0,0 +1,78 @@
+FROM debian:bookworm-slim
+
+# Avoid prompts from apt
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install base packages for GIS tools
+RUN apt-get update && apt-get install -y \
+    bash \
+    curl \
+    wget \
+    git \
+    python3 \
+    python3-pip \
+    build-essential \
+    sudo \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create symbolic link for python
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+# Install GIS tools
+RUN apt-get update && apt-get install -y \
+    gdal-bin \
+    libgdal-dev \
+    proj-bin \
+    proj-data \
+    libproj-dev \
+    postgis \
+    postgresql-client \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install DuckDB with spatial extensions
+RUN curl -L https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip \
+    -o /tmp/duckdb.zip && \
+    cd /tmp && \
+    unzip duckdb.zip && \
+    cp duckdb_*_linux_amd64 /usr/local/bin/duckdb && \
+    chmod +x /usr/local/bin/duckdb && \
+    rm -rf /tmp/duckdb*
+
+# Install Python GIS libraries
+RUN pip3 install --break-system-packages \
+    geopandas \
+    shapely \
+    rasterio \
+    folium \
+    plotly \
+    xarray \
+    cfgrib \
+    netcdf4 \
+    matplotlib \
+    seaborn \
+    duckdb \
+    dask
+
+# Install R with spatial packages (if needed)
+RUN apt-get update && apt-get install -y \
+    r-base \
+    r-base-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install additional tools for weather data processing
+RUN apt-get update && apt-get install -y \
+    ftp \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install R spatial packages
+RUN R --slave -e "install.packages(c('raster', 'rgdal', 'ncdf4', 'rasterVis', 'leaflet'), repos='https://cran.rstudio.com/', dependencies=TRUE)"
+
+# Add entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Create a working directory
+WORKDIR /workspace
+
+# Use the entrypoint script to handle user creation
+ENTRYPOINT ["/entrypoint.sh"]
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/README.md
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/README.md
@@ -0,0 +1,129 @@
+# TSYS-AIOS-GIS-Tools-GIS-Base Container
+
+This container is part of the TSYS-AIOS-GIS project and provides a base GIS and weather data processing environment.
+
+## Overview
+
+The TSYS-AIOS-GIS-Tools-GIS-Base container is designed for GIS data processing and weather data analysis tasks. It includes essential tools for handling geospatial data formats and weather datasets with a focus on self-hosted GIS stack capabilities.
+
+## Tools Included
+
+### Core Tools
+- **Base OS**: Debian Bookworm slim
+- **Shell**: Bash
+- **Programming Languages**:
+  - Python 3 with geospatial libraries
+  - R with spatial packages
+
+### GIS Libraries
+- **GDAL/OGR**: Geospatial Data Abstraction Library for format translation and processing
+- **PROJ**: Coordinate transformation software
+- **PostGIS**: Client tools for spatial database operations
+- **DuckDB**: With spatial extensions for efficient data processing
+- **GeoPandas**: Python geospatial data handling
+- **Shapely**: Python geometric operations
+- **Rasterio**: Raster processing in Python
+
+### Weather Data Processing
+- **xarray**: Multi-dimensional data in Python
+- **cfgrib**: GRIB format handling
+- **netCDF4**: NetCDF file handling
+- **MetPy**: Meteorological calculations (via Python libraries)
+
+### Visualization
+- **Folium**: Interactive maps
+- **Plotly**: Time series visualization
+- **Matplotlib/Seaborn**: Statistical plots
+- **R visualization packages**: For statistical analysis
+
+### Additional Tools
+- **Dask**: For large data processing
+- **FTP client**: For bulk data downloads
+
+## Usage
+
+### Building the Base Container
+```bash
+# From this directory
+cd /home/localuser/AIWorkspace/TSYS-AIOS-GIS/Docker/TSYS-AIOS-GIS-Tools-GIS-Base
+
+# Use the wrapper script to automatically detect and set user IDs
+./docker-compose-wrapper.sh build
+
+# Or run commands in the base container with automatic user mapping
+./docker-compose-wrapper.sh run tsys-gis-base [command]
+
+# Example: Process a shapefile with GDAL
+./docker-compose-wrapper.sh run tsys-gis-base ogrinfo /workspace/path/to/shapefile.shp
+
+# Example: Start Python with geospatial libraries
+./docker-compose-wrapper.sh run tsys-gis-base python3
+
+# Example: Start R with spatial packages
+./docker-compose-wrapper.sh run tsys-gis-base R
+```
+
+### Using with docker-compose directly
+```bash
+# Set environment variables and run docker-compose directly
+LOCAL_USER_ID=$(id -u) LOCAL_GROUP_ID=$(id -g) docker-compose up --build
+
+# Or export variables first
+export LOCAL_USER_ID=$(id -u)
+export LOCAL_GROUP_ID=$(id -g) 
+docker-compose up
+```
+
+### Using the wrapper script
+```bash
+# Build and start the base GIS container with automatic user mapping
+./docker-compose-wrapper.sh up --build
+
+# Start without rebuilding
+./docker-compose-wrapper.sh up
+
+# View container status
+./docker-compose-wrapper.sh ps
+
+# Stop containers
+./docker-compose-wrapper.sh down
+```
+
+## User ID Mapping (For File Permissions)
+
+The container automatically detects and uses the host user's UID and GID to ensure proper file permissions. This means:
+
+- Files created inside the container will have the correct ownership on the host
+- No more root-owned files after container operations
+- Works across different environments (development, production servers)
+
+The container detects the user ID from the mounted workspace volume. If needed, you can override the default values by setting environment variables:
+
+```bash
+# Set specific user ID and group ID before running docker-compose
+export LOCAL_USER_ID=1000
+export LOCAL_GROUP_ID=1000
+docker-compose up
+```
+
+Or run with inline environment variables:
+```bash
+LOCAL_USER_ID=1000 LOCAL_GROUP_ID=1000 docker-compose up
+```
+
+The container runs as a non-root user named `TSYS-Tools` with the detected host user's UID/GID.
+
+## Data Processing Workflows
+
+This container is designed to handle both:
+- GIS data processing (shapefiles, GeoJSON, Parquet, etc.)
+- Weather data processing (GRIB, NetCDF formats)
+- ETL workflows for geospatial and meteorological datasets
+- Integration with PostGIS for spatial database operations
+- Output to MinIO buckets for business use
+
+## Integration
+
+- Can be used in CTO mode for R&D activities
+- Compatible with existing documentation containers for report generation
+- Designed for both workstation prototyping and server deployment
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/docker-compose-wrapper.sh
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/docker-compose-wrapper.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+# docker-compose-wrapper.sh - Wrapper script to detect host UID/GID and run docker-compose
+
+set -e  # Exit on any error
+
+# Detect the UID and GID of the user that owns the workspace directory (parent directory)
+WORKSPACE_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
+echo "Detecting user ID from workspace directory: $WORKSPACE_DIR"
+
+if [ -d "$WORKSPACE_DIR" ]; then
+    DETECTED_USER_ID=$(stat -c %u "$WORKSPACE_DIR" 2>/dev/null || echo 0)
+    DETECTED_GROUP_ID=$(stat -c %g "$WORKSPACE_DIR" 2>/dev/null || echo 0)
+    
+    # If detection failed, try current user
+    if [ "$DETECTED_USER_ID" = "0" ]; then
+        DETECTED_USER_ID=$(id -u)
+        DETECTED_GROUP_ID=$(id -g)
+    fi
+else
+    # Fallback to current user if workspace directory doesn't exist
+    DETECTED_USER_ID=$(id -u)
+    DETECTED_GROUP_ID=$(id -g)
+fi
+
+echo "Detected USER_ID=$DETECTED_USER_ID and GROUP_ID=$DETECTED_GROUP_ID"
+
+# Set environment variables for docker-compose
+export LOCAL_USER_ID=$DETECTED_USER_ID
+export LOCAL_GROUP_ID=$DETECTED_GROUP_ID
+
+# Show usage information
+echo ""
+echo "Usage: $0 [build|up|run <service> <command>|exec <service> <command>|down|ps]"
+echo ""
+echo "Examples:"
+echo "  $0 up                    # Start services"
+echo "  $0 build                 # Build containers"
+echo "  $0 run tsys-gis-base bash  # Run command in container"
+echo "  $0 down                  # Stop and remove containers"
+echo ""
+
+# Check if docker compose (new format) or docker-compose (old format) is available
+if command -v docker &> /dev/null && docker compose version &> /dev/null; then
+    # Use new docker compose format
+    if [ $# -eq 0 ]; then
+        echo "No command provided. Running 'docker compose up'..."
+        docker compose up
+    else
+        # Execute the provided docker compose command
+        echo "Running: docker compose $*"
+        docker compose "$@"
+    fi
+elif command -v docker-compose &> /dev/null; then
+    # Fallback to old docker-compose format
+    if [ $# -eq 0 ]; then
+        echo "No command provided. Running 'docker-compose up'..."
+        docker-compose up
+    else
+        # Execute the provided docker-compose command
+        echo "Running: docker-compose $*"
+        docker-compose "$@"
+    fi
+else
+    echo "Error: Neither 'docker compose' nor 'docker-compose' command found."
+    echo "Please install Docker Compose to use this script."
+    exit 1
+fi
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/docker-compose.yml
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/docker-compose.yml
@@ -0,0 +1,18 @@
+version: '3.8'
+
+services:
+  tsys-gis-base:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: TSYS-AIOS-GIS-Tools-GIS-Base
+    image: tsys-aios-gis-tools-gis-base:latest
+    volumes:
+      - ../../../:/workspace:rw
+    working_dir: /workspace
+    stdin_open: true
+    tty: true
+    environment:
+      - LOCAL_USER_ID=${LOCAL_USER_ID:-1000}
+      - LOCAL_GROUP_ID=${LOCAL_GROUP_ID:-1000}
+    user: "${LOCAL_USER_ID:-1000}:${LOCAL_GROUP_ID:-1000}"
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/entrypoint.sh
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Base/entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# entrypoint.sh - Entrypoint script to handle user creation and permission setup at runtime
+
+# Set default values if not provided
+USER_ID=${LOCAL_USER_ID:-$(id -u 1000)}
+GROUP_ID=${LOCAL_GROUP_ID:-$(id -g 1000)}
+
+# In case the environment variables are not set properly, detect them from the workspace volume
+if [ "$USER_ID" = "$(id -u 0)" ] || [ "$USER_ID" = "0" ]; then
+    # Detect the UID and GID of the user that owns the workspace directory
+    if [ -d "/workspace" ]; then
+        USER_ID=$(stat -c %u /workspace 2>/dev/null || echo 1000)
+        GROUP_ID=$(stat -c %g /workspace 2>/dev/null || echo 1000)
+    else
+        USER_ID=${LOCAL_USER_ID:-1000}
+        GROUP_ID=${LOCAL_GROUP_ID:-1000}
+    fi
+fi
+
+echo "Starting with USER_ID=$USER_ID and GROUP_ID=$GROUP_ID"
+
+# Create the group with specified GID
+groupadd -f -g $GROUP_ID -o TSYS-Tools 2>/dev/null || groupmod -g $GROUP_ID -o TSYS-Tools
+
+# Create the user with specified UID and add to the group
+useradd -f -u $USER_ID -g $GROUP_ID -m -s /bin/bash -o TSYS-Tools 2>/dev/null || usermod -u $USER_ID -g $GROUP_ID -o TSYS-Tools
+
+# Add user to sudo group for any necessary operations
+usermod -aG sudo TSYS-Tools 2>/dev/null || true
+
+# Make sure workspace directory exists and has proper permissions
+mkdir -p /workspace
+chown -R $USER_ID:$GROUP_ID /workspace
+
+# Set up proper permissions for .local (if they exist)
+mkdir -p /home/TSYS-Tools/.local
+chown $USER_ID:$GROUP_ID /home/TSYS-Tools/.local
+
+# Set up proper permissions for R (if they exist)
+mkdir -p /home/TSYS-Tools/R
+chown $USER_ID:$GROUP_ID /home/TSYS-Tools/R
+
+# If there are additional arguments, run them as the created user
+if [ $# -gt 0 ]; then
+    exec su -p TSYS-Tools -c "$*"
+else
+    # Otherwise start an interactive bash shell as the created user
+    exec su -p TSYS-Tools -c "/bin/bash"
+fi