Add core architecture patterns and GIS/weather components from AIOS-Public

2025-10-16 13:14:30 -05:00
parent 782eec63a5
commit 5887f4e729
32 changed files with 1970 additions and 1 deletions
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/Dockerfile
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/Dockerfile
@@ -0,0 +1,32 @@
+FROM tsys-aios-gis-tools-gis-base:latest
+
+# Avoid prompts from apt
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install additional processing tools
+RUN apt-get update && apt-get install -y \
+    jupyter-notebook \
+    nodejs \
+    npm \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install additional Python libraries for processing
+RUN pip3 install --break-system-packages \
+    jupyter \
+    notebook \
+    ipykernel \
+    apache-airflow \
+    prefect
+
+# Set up Jupyter
+RUN jupyter notebook --generate-config --allow-root --ip=0.0.0.0 --port=8888 --no-browser --notebook-dir=/workspace --NotebookApp.token='' --NotebookApp.password='' 2>/dev/null || true
+
+# Create a working directory
+WORKDIR /workspace
+
+# Add entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Use the entrypoint script to handle user creation
+ENTRYPOINT ["/entrypoint.sh"]
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/README.md
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/README.md
@@ -0,0 +1,111 @@
+# TSYS-AIOS-GIS-Tools-GIS-Processing Container
+
+This container is part of the TSYS-AIOS-GIS project and provides advanced GIS data processing capabilities with Jupyter notebooks and workflow tools.
+
+## Overview
+
+The TSYS-AIOS-GIS-Tools-GIS-Processing container extends the base GIS container with advanced processing tools, Jupyter notebooks for interactive analysis, and workflow orchestration tools. This container is designed for in-depth geospatial data analysis and complex ETL workflows.
+
+## Tools Included
+
+### Extends from Base Container
+- All tools from TSYS-AIOS-GIS-Tools-GIS-Base container
+- GIS libraries, weather data processing libraries, visualization tools
+
+### Advanced Processing Tools
+- **Jupyter Notebook**: Interactive environment for data analysis
+- **Node.js/npm**: JavaScript runtime and package manager
+- **IPyKernel**: IPython kernel for Jupyter
+
+### Workflow Tools
+- **Apache Airflow**: Workflow orchestration platform
+- **Prefect**: Modern workflow management
+
+## Usage
+
+### Building the Processing Container
+```bash
+# From this directory
+cd /home/localuser/AIWorkspace/TSYS-AIOS-GIS/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing
+
+# Use the wrapper script to automatically detect and set user IDs
+./docker-compose-wrapper.sh build
+
+# Or run commands in the processing container with automatic user mapping
+./docker-compose-wrapper.sh run tsys-gis-processing [command]
+
+# Example: Start Jupyter notebook server
+./docker-compose-wrapper.sh run tsys-gis-processing jupyter notebook --ip=0.0.0.0 --port=8888 --allow-root --notebook-dir=/workspace --no-browser
+
+# Example: Start an interactive bash session
+./docker-compose-wrapper.sh run tsys-gis-processing bash
+```
+
+### Using with docker-compose directly
+```bash
+# Set environment variables and run docker-compose directly
+LOCAL_USER_ID=$(id -u) LOCAL_GROUP_ID=$(id -g) docker-compose up --build
+
+# Or export variables first
+export LOCAL_USER_ID=$(id -u)
+export LOCAL_GROUP_ID=$(id -g) 
+docker-compose up
+```
+
+### Using the wrapper script
+```bash
+# Build and start the processing container with automatic user mapping
+./docker-compose-wrapper.sh up --build
+
+# Start without rebuilding (Jupyter will be available on port 8888)
+./docker-compose-wrapper.sh up
+
+# View container status
+./docker-compose-wrapper.sh ps
+
+# Stop containers
+./docker-compose-wrapper.sh down
+```
+
+## Jupyter Notebook Access
+
+When running the container with `docker-compose up`, Jupyter notebook will be available at:
+- http://localhost:8888
+
+The notebook server is preconfigured to:
+- Use the workspace directory as the notebook directory
+- Allow access without authentication (in container only)
+- Accept connections from any IP address
+
+## User ID Mapping (For File Permissions)
+
+The container automatically detects and uses the host user's UID and GID to ensure proper file permissions. This means:
+
+- Files created inside the container will have the correct ownership on the host
+- No more root-owned files after container operations
+- Works across different environments (development, production servers)
+
+The container detects the user ID from the mounted workspace volume. If needed, you can override the default values by setting environment variables:
+
+```bash
+# Set specific user ID and group ID before running docker-compose
+export LOCAL_USER_ID=1000
+export LOCAL_GROUP_ID=1000
+docker-compose up
+```
+
+Or run with inline environment variables:
+```bash
+LOCAL_USER_ID=1000 LOCAL_GROUP_ID=1000 docker-compose up
+```
+
+The container runs as a non-root user named `TSYS-Tools` with the detected host user's UID/GID.
+
+## Data Processing Workflows
+
+This container is optimized for:
+- Interactive geospatial analysis using Jupyter notebooks
+- Complex ETL workflows using Apache Airflow or Prefect
+- Advanced visualization and reporting
+- Model development and testing
+- Integration with PostGIS and other databases
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/docker-compose-wrapper.sh
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/docker-compose-wrapper.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+# docker-compose-wrapper.sh - Wrapper script to detect host UID/GID and run docker-compose
+
+set -e  # Exit on any error
+
+# Detect the UID and GID of the user that owns the workspace directory (parent directory)
+WORKSPACE_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
+echo "Detecting user ID from workspace directory: $WORKSPACE_DIR"
+
+if [ -d "$WORKSPACE_DIR" ]; then
+    DETECTED_USER_ID=$(stat -c %u "$WORKSPACE_DIR" 2>/dev/null || echo 0)
+    DETECTED_GROUP_ID=$(stat -c %g "$WORKSPACE_DIR" 2>/dev/null || echo 0)
+    
+    # If detection failed, try current user
+    if [ "$DETECTED_USER_ID" = "0" ]; then
+        DETECTED_USER_ID=$(id -u)
+        DETECTED_GROUP_ID=$(id -g)
+    fi
+else
+    # Fallback to current user if workspace directory doesn't exist
+    DETECTED_USER_ID=$(id -u)
+    DETECTED_GROUP_ID=$(id -g)
+fi
+
+echo "Detected USER_ID=$DETECTED_USER_ID and GROUP_ID=$DETECTED_GROUP_ID"
+
+# Set environment variables for docker-compose
+export LOCAL_USER_ID=$DETECTED_USER_ID
+export LOCAL_GROUP_ID=$DETECTED_GROUP_ID
+
+# Show usage information
+echo ""
+echo "Usage: $0 [build|up|run <service> <command>|exec <service> <command>|down|ps]"
+echo ""
+echo "Examples:"
+echo "  $0 up                    # Start services"
+echo "  $0 build                 # Build containers"
+echo "  $0 run tsys-gis-processing bash  # Run command in container"
+echo "  $0 down                  # Stop and remove containers"
+echo ""
+
+# Check if docker compose (new format) or docker-compose (old format) is available
+if command -v docker &> /dev/null && docker compose version &> /dev/null; then
+    # Use new docker compose format
+    if [ $# -eq 0 ]; then
+        echo "No command provided. Running 'docker compose up'..."
+        docker compose up
+    else
+        # Execute the provided docker compose command
+        echo "Running: docker compose $*"
+        docker compose "$@"
+    fi
+elif command -v docker-compose &> /dev/null; then
+    # Fallback to old docker-compose format
+    if [ $# -eq 0 ]; then
+        echo "No command provided. Running 'docker-compose up'..."
+        docker-compose up
+    else
+        # Execute the provided docker-compose command
+        echo "Running: docker-compose $*"
+        docker-compose "$@"
+    fi
+else
+    echo "Error: Neither 'docker compose' nor 'docker-compose' command found."
+    echo "Please install Docker Compose to use this script."
+    exit 1
+fi
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/docker-compose.yml
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/docker-compose.yml
@@ -0,0 +1,20 @@
+version: '3.8'
+
+services:
+  tsys-gis-processing:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: TSYS-AIOS-GIS-Tools-GIS-Processing
+    image: tsys-aios-gis-tools-gis-processing:latest
+    volumes:
+      - ../../../:/workspace:rw
+    working_dir: /workspace
+    stdin_open: true
+    tty: true
+    ports:
+      - "8888:8888"
+    environment:
+      - LOCAL_USER_ID=${LOCAL_USER_ID:-1000}
+      - LOCAL_GROUP_ID=${LOCAL_GROUP_ID:-1000}
+    user: "${LOCAL_USER_ID:-1000}:${LOCAL_GROUP_ID:-1000}"
--- a/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/entrypoint.sh
+++ b/Docker/TSYS-AIOS-GIS-Tools-GIS-Processing/entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# entrypoint.sh - Entrypoint script to handle user creation and permission setup at runtime
+
+# Set default values if not provided
+USER_ID=${LOCAL_USER_ID:-$(id -u 1000)}
+GROUP_ID=${LOCAL_GROUP_ID:-$(id -g 1000)}
+
+# In case the environment variables are not set properly, detect them from the workspace volume
+if [ "$USER_ID" = "$(id -u 0)" ] || [ "$USER_ID" = "0" ]; then
+    # Detect the UID and GID of the user that owns the workspace directory
+    if [ -d "/workspace" ]; then
+        USER_ID=$(stat -c %u /workspace 2>/dev/null || echo 1000)
+        GROUP_ID=$(stat -c %g /workspace 2>/dev/null || echo 1000)
+    else
+        USER_ID=${LOCAL_USER_ID:-1000}
+        GROUP_ID=${LOCAL_GROUP_ID:-1000}
+    fi
+fi
+
+echo "Starting with USER_ID=$USER_ID and GROUP_ID=$GROUP_ID"
+
+# Create the group with specified GID
+groupadd -f -g $GROUP_ID -o TSYS-Tools 2>/dev/null || groupmod -g $GROUP_ID -o TSYS-Tools
+
+# Create the user with specified UID and add to the group
+useradd -f -u $USER_ID -g $GROUP_ID -m -s /bin/bash -o TSYS-Tools 2>/dev/null || usermod -u $USER_ID -g $GROUP_ID -o TSYS-Tools
+
+# Add user to sudo group for any necessary operations
+usermod -aG sudo TSYS-Tools 2>/dev/null || true
+
+# Make sure workspace directory exists and has proper permissions
+mkdir -p /workspace
+chown -R $USER_ID:$GROUP_ID /workspace
+
+# Set up proper permissions for .local (if they exist)
+mkdir -p /home/TSYS-Tools/.local
+chown $USER_ID:$GROUP_ID /home/TSYS-Tools/.local
+
+# Set up proper permissions for Jupyter (if they exist)
+mkdir -p /home/TSYS-Tools/.jupyter
+chown $USER_ID:$GROUP_ID /home/TSYS-Tools/.jupyter
+
+# If there are additional arguments, run them as the created user
+if [ $# -gt 0 ]; then
+    exec su -p TSYS-Tools -c "$*"
+else
+    # Otherwise start an interactive bash shell as the created user
+    exec su -p TSYS-Tools -c "/bin/bash"
+fi