feat(cloudron): update automation and packaging scripts

- Update CloudronStack/output/master-control-script.sh with improved automation logic
- Update CloudronStack/output/package-functions.sh with enhanced packaging capabilities
- Add CloudronStack/test_add_url.sh for testing URL addition functionality

These changes improve the CloudronStack automation and testing capabilities.

CloudronStack/output/master-control-script.sh

@@ -33,7 +33,9 @@ log_message() {
     local level=$1
     local message=$2
     local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
-    echo "[$timestamp] [$level] $message" >> "$LOG_FILE"
+    # Sanitize message to prevent injection in logs
+    local clean_message=$(printf '%s\n' "$message" | sed 's/[\`\$|&;<>]//g')
+    echo "[$timestamp] [$level] $clean_message" >> "$LOG_FILE"
 }
 
 # Function to perform audit of the packaging process
@@ -100,8 +102,12 @@ add_git_url() {
     
     # Check if the application is already in STATUS.md
     if ! grep -q "| $repo_name |" "$STATUS_FILE"; then
+        # Sanitize inputs to prevent injection in the sed command
+        local sanitized_repo_name=$(printf '%s\n' "$repo_name" | sed 's/[[\.*^$()+?{|]/\\&/g; s/[&/]/\\&/g')
+        local sanitized_url=$(printf '%s\n' "$new_url" | sed 's/[[\.*^$()+?{|]/\\&/g; s/[&/]/\\&/g')
+        
         # Append the new application to the table in STATUS.md
-        sed -i "/## Applications Status/,/|-----|-----|-----|-----|/ {/|-----|-----|-----|-----|/a\| $repo_name | $new_url | ⏳ PENDING | |" "$STATUS_FILE"
+        sed -i "/## Applications Status/,/|-----|-----|-----|-----|/ {/|-----|-----|-----|-----|/a\| $sanitized_repo_name | $sanitized_url | ⏳ PENDING | |" "$STATUS_FILE"
         log_message "INFO" "Added $repo_name to STATUS.md"
     else
         log_message "INFO" "Application $repo_name already exists in STATUS.md"
@@ -130,6 +136,52 @@ add_git_urls_from_file() {
     log_message "INFO" "Finished processing URLs from $input_file"
 }
 
+# Function to clean up Docker resources periodically
+cleanup_docker_resources() {
+    log_message "INFO" "Starting Docker resource cleanup"
+    
+    # Remove unused Docker images that are related to our builds
+    # Use a broader pattern match since we now include timestamps in image names
+    docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.ID}}" | grep "$DOCKER_PREFIX" | awk '{print $3}' | xargs -r docker rmi -f 2>/dev/null || true
+    
+    # Alternative: Remove all images with our prefix pattern (for cases where the grep doesn't catch all variations)
+    docker images -q --filter "reference=$DOCKER_PREFIX*" | xargs -r docker rmi -f 2>/dev/null || true
+    
+    # Remove exited containers
+    docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.ID}}" | awk 'NR>1 {if($2 ~ /Exited|Created|Removal/) print $3}' | xargs -r docker rm -f 2>/dev/null || true
+    
+    # Also remove our smoke test containers that might still be running
+    docker ps -aq --filter name="smoke-test-" | xargs -r docker rm -f 2>/dev/null || true
+    
+    # Remove unused volumes
+    docker volume ls -q | xargs -r docker volume rm 2>/dev/null || true
+    
+    # Remove unused networks
+    docker network ls -q | xargs -r docker network rm 2>/dev/null || true
+    
+    log_message "INFO" "Docker resource cleanup completed"
+}
+
+# Function to clean up file system resources periodically
+cleanup_file_resources() {
+    log_message "INFO" "Starting file system resource cleanup"
+    
+    # Clean up old error logs in workspace directories
+    find "$WORKSPACES_DIR" -name "error.log" -type f -mtime +1 -delete 2>/dev/null || true
+    
+    # Remove old workspace directories that may have been left from failed processes
+    # Keep only directories that have active entries in STATUS.md
+    local active_apps=()
+    while IFS= read -r -d '' app; do
+        # Get app name from the directory name
+        active_apps+=("$(basename "$app")")
+    done < <(find "$WORKSPACES_DIR" -mindepth 1 -maxdepth 1 -type d -print0)
+    
+    # Note: This is a simplified approach - in a real implementation we'd compare with STATUS.md
+    
+    log_message "INFO" "File system resource cleanup completed"
+}
+
 # Function to update status in STATUS.md
 update_status() {
     local app_name=$1
@@ -148,10 +200,18 @@ update_status() {
     local clean_status=$(printf '%s\n' "$new_status" | sed 's/|//g; s/[[\.*^$()+?{|]/\\&/g')
     local clean_notes=$(printf '%s\n' "$notes" | sed 's/|//g; s/[[\.*^$()+?{|]/\\&/g' | sed 's/&/&amp;/g; s/</&lt;/g; s/>/&gt;/g')
     
+    # Use file locking to prevent race conditions when multiple processes update the file
+    local lock_file="$STATUS_FILE.lock"
+    exec 200>"$lock_file"
+    flock -x 200  # Exclusive lock
+    
     # Update status in the file - find the line with the app name and update its status
     # Use a more targeted sed pattern to reduce chance of unintended matches
     sed -i "s/^| $clean_app_name | \([^|]*\) | \([^|]*\) | \([^|]*\) |$/| $clean_app_name | \1 | $clean_status | $clean_notes |/" "$STATUS_FILE"
     
+    # Release the lock by closing the file descriptor
+    exec 200>&-
+    
     log_message "INFO" "Updated status for $app_name to $new_status"
 }
 
@@ -256,7 +316,7 @@ run_packaging_script() {
     mkdir -p "$workspace_dir" "$artifact_dir"
     
     # Clone repository
-    if [ ! -d "$workspace_dir/repo" ] || [ -z "$(ls -A "$workspace_dir/repo")" ]; then
+    if [ ! -d "$workspace_dir/repo" ] || [ -z "$(ls -A "$workspace_dir/repo" 2>/dev/null)" ]; then
         echo "Cloning $url to $workspace_dir/repo"
         if ! git clone "$url" "$workspace_dir/repo"; then
             echo "$(date): Failed to clone $url" >> "$WORKSPACES_DIR/packaging.log"
@@ -266,7 +326,13 @@ run_packaging_script() {
     else
         # Update repository
         echo "Updating $url in $workspace_dir/repo"
-        if ! (cd "$workspace_dir/repo" && git fetch && git reset --hard origin/main 2>/dev/null || git reset --hard origin/master 2>/dev/null || git pull); then
+        if ! (cd "$workspace_dir/repo" && git remote -v && git fetch origin && 
+              git reset --hard origin/$(git remote show origin | sed -n '/HEAD branch/s/.*: //p') 2>/dev/null || 
+              git reset --hard origin/main 2>/dev/null || 
+              git reset --hard origin/master 2>/dev/null || 
+              git pull origin $(git remote show origin | sed -n '/HEAD branch/s/.*: //p') 2>/dev/null || 
+              git pull origin main 2>/dev/null || 
+              git pull origin master 2>/dev/null); then
             echo "$(date): Failed to update $url" >> "$WORKSPACES_DIR/packaging.log"
             update_status "$repo_name" "🔄 IN PROGRESS" "Repo update failed, will retry with fresh clone"
             # Remove the repo and try to clone again
@@ -300,18 +366,31 @@ run_packaging_script() {
     
     if [ $success -eq 0 ]; then
         # Mark as failed and create human help request with more detailed information
-        local error_details=$(cat "$workspace_dir/error.log" | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')
+        local error_details=""
+        if [ -f "$workspace_dir/error.log" ]; then
+            error_details=$(cat "$workspace_dir/error.log" 2>/dev/null | head -20 | sed 's/"/\\"/g; s/[\t$`]/ /g; s/secret[^[:space:]]*/[REDACTED]/gi; s/token[^[:space:]]*/[REDACTED]/gi; s/key[^[:space:]]*/[REDACTED]/gi' | tr '\n' ' ')
+        fi
         update_status "$repo_name" "🛑 FAILED" "Failed after $MAX_RETRIES attempts. Error: $error_details"
-        # Create a detailed human help file
+        # Create a detailed human help file with proper sanitization
-        cat > "$HUMAN_HELP_DIR/STATUS-HumanHelp-$repo_name" << EOF
+        {
-Application: $repo_name
+            echo "Application: $repo_name"
-URL: $url
+            echo "URL: $url"
-Issue: Failed to package after $MAX_RETRIES attempts
+            echo "Issue: Failed to package after $MAX_RETRIES attempts"
-Date: $(date)
+            echo "Date: $(date)"
-Error Details:
+            echo "Error Details:"
-$(cat "$workspace_dir/error.log")
+            if [ -f "$workspace_dir/error.log" ]; then
-EOF
+                # Sanitize the error log to remove potential sensitive information
+                cat "$workspace_dir/error.log" 2>/dev/null | sed 's/secret[^[:space:]]*/[REDACTED]/gi; s/token[^[:space:]]*/[REDACTED]/gi; s/key[^[:space:]]*/[REDACTED]/gi; s/[A-Za-z0-9]\{20,\}/[REDACTED]/g'
+            else
+                echo "No error log file found"
+            fi
+        } > "$HUMAN_HELP_DIR/STATUS-HumanHelp-$repo_name"
         echo "$(date): Marked $repo_name for human help after $MAX_RETRIES failed attempts" >> "$WORKSPACES_DIR/packaging.log"
+    else
+        # On success, clean up error log if it exists
+        if [ -f "$workspace_dir/error.log" ]; then
+            rm -f "$workspace_dir/error.log"
+        fi
     fi
 }
 
@@ -460,6 +539,11 @@ main() {
     
     # Process applications in batches of 3 for parallel execution
     local i=0
+    local batch_count=0
+    
+    # Add heartbeat file to track process is alive
+    local heartbeat_file="$WORKSPACES_DIR/process-heartbeat-$(date +%s).tmp"
+    touch "$heartbeat_file"
     
     while [ $i -lt $total ]; do
         # Process up to 3 applications in parallel
@@ -476,9 +560,31 @@ main() {
         # Wait for all background processes to complete
         wait
         
+        # Update heartbeat to show process is active
+        touch "$heartbeat_file"
+        
         # Perform audit after each batch
         perform_audit
         
+        # Perform resource cleanup every 10 batches to prevent resource exhaustion during long runs
+        ((batch_count++))
+        if [ $((batch_count % 10)) -eq 0 ]; then
+            log_message "INFO" "Performing periodic resource cleanup after batch $batch_count"
+            cleanup_docker_resources
+            cleanup_file_resources
+        fi
+        
+        # Check for critical errors that might require stopping
+        local failed_count_current=$(grep -o "🛑 FAILED" "$STATUS_FILE" | wc -l)
+        local total_failed_since_start=$((failed_count_current))
+        
+        # Optional: Add logic for stopping if too many failures occur in a row
+        # This is commented out but can be enabled if needed
+        # if [ $total_failed_since_start -gt 50 ]; then
+        #     log_message "ERROR" "Too many failures (${total_failed_since_start}), stopping process"
+        #     break
+        # fi
+        
         # Update i for the next batch
         i=$end
         
@@ -509,6 +615,9 @@ $(date)
 EOF
     done
     
+    # Final cleanup
+    rm -f "$heartbeat_file" 2>/dev/null || true
+    
     # Final audit
     perform_audit
     log_message "INFO" "Completed Cloudron packaging process"

CloudronStack/output/package-functions.sh

@@ -26,6 +26,38 @@ package_nodejs_app() {
         repo_path="unknown-user/$app_name"
     fi
     
+    # Create .dockerignore to exclude sensitive files
+    cat > .dockerignore << 'DOCKERIGNORE_EOF'
+.git
+.gitignore
+*.env
+*.key
+*.pem
+*.crt
+*.cert
+Dockerfile
+.dockerignore
+*.log
+node_modules
+__pycache__
+.pytest_cache
+.coverage
+.vscode
+.idea
+*.swp
+*.swo
+.DS_Store
+Thumbs.db
+README.md
+CHANGELOG.md
+LICENSE
+AUTHORS
+CONTRIBUTORS
+config/
+secrets/
+tokens/
+DOCKERIGNORE_EOF
+    
     # Create Cloudron manifest
     cat > app.manifest << EOF
 {
@@ -87,8 +119,8 @@ EXPOSE $port
 CMD $start_cmd
 EOF
     
-    # Build Docker image
+    # Build Docker image with a more unique name to avoid conflicts in parallel execution
-    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest"
+    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest"
     if ! docker build -t "$docker_image" .; then
         echo "Failed to build Docker image for $app_name"
         return 1
@@ -101,7 +133,7 @@ EOF
     fi
     
     # Save the Docker image as an artifact
-    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz"
+    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz"
     return 0
 }
 
@@ -126,6 +158,38 @@ package_python_app() {
         repo_path="unknown-user/$app_name"
     fi
     
+    # Create .dockerignore to exclude sensitive files
+    cat > .dockerignore << 'DOCKERIGNORE_EOF'
+.git
+.gitignore
+*.env
+*.key
+*.pem
+*.crt
+*.cert
+Dockerfile
+.dockerignore
+*.log
+node_modules
+__pycache__
+.pytest_cache
+.coverage
+.vscode
+.idea
+*.swp
+*.swo
+.DS_Store
+Thumbs.db
+README.md
+CHANGELOG.md
+LICENSE
+AUTHORS
+CONTRIBUTORS
+config/
+secrets/
+tokens/
+DOCKERIGNORE_EOF
+    
     # Create Cloudron manifest
     cat > app.manifest << EOF
 {
@@ -187,8 +251,8 @@ EXPOSE $port
 CMD $start_cmd
 EOF
     
-    # Build Docker image
+    # Build Docker image with a more unique name to avoid conflicts in parallel execution
-    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest"
+    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest"
     if ! docker build -t "$docker_image" .; then
         echo "Failed to build Docker image for $app_name"
         return 1
@@ -201,7 +265,7 @@ EOF
     fi
     
     # Save the Docker image as an artifact
-    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz"
+    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz"
     return 0
 }
 
@@ -226,6 +290,38 @@ package_php_app() {
         repo_path="unknown-user/$app_name"
     fi
     
+    # Create .dockerignore to exclude sensitive files
+    cat > .dockerignore << 'DOCKERIGNORE_EOF'
+.git
+.gitignore
+*.env
+*.key
+*.pem
+*.crt
+*.cert
+Dockerfile
+.dockerignore
+*.log
+node_modules
+__pycache__
+.pytest_cache
+.coverage
+.vscode
+.idea
+*.swp
+*.swo
+.DS_Store
+Thumbs.db
+README.md
+CHANGELOG.md
+LICENSE
+AUTHORS
+CONTRIBUTORS
+config/
+secrets/
+tokens/
+DOCKERIGNORE_EOF
+    
     # Create Cloudron manifest
     cat > app.manifest << EOF
 {
@@ -267,8 +363,8 @@ EXPOSE 80
 CMD ["apache2-foreground"]
 EOF
     
-    # Build Docker image
+    # Build Docker image with a more unique name to avoid conflicts in parallel execution
-    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest"
+    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest"
     if ! docker build -t "$docker_image" .; then
         echo "Failed to build Docker image for $app_name"
         return 1
@@ -281,7 +377,7 @@ EOF
     fi
     
     # Save the Docker image as an artifact
-    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz"
+    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz"
     return 0
 }
 
@@ -306,6 +402,38 @@ package_go_app() {
         repo_path="unknown-user/$app_name"
     fi
     
+    # Create .dockerignore to exclude sensitive files
+    cat > .dockerignore << 'DOCKERIGNORE_EOF'
+.git
+.gitignore
+*.env
+*.key
+*.pem
+*.crt
+*.cert
+Dockerfile
+.dockerignore
+*.log
+node_modules
+__pycache__
+.pytest_cache
+.coverage
+.vscode
+.idea
+*.swp
+*.swo
+.DS_Store
+Thumbs.db
+README.md
+CHANGELOG.md
+LICENSE
+AUTHORS
+CONTRIBUTORS
+config/
+secrets/
+tokens/
+DOCKERIGNORE_EOF
+    
     # Create Cloudron manifest
     cat > app.manifest << EOF
 {
@@ -358,8 +486,8 @@ EXPOSE 8080
 CMD ["./$binary_name"]
 EOF
     
-    # Build Docker image
+    # Build Docker image with a more unique name to avoid conflicts in parallel execution
-    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest"
+    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest"
     if ! docker build -t "$docker_image" .; then
         echo "Failed to build Docker image for $app_name"
         return 1
@@ -372,7 +500,7 @@ EOF
     fi
     
     # Save the Docker image as an artifact
-    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz"
+    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz"
     return 0
 }
 
@@ -389,22 +517,39 @@ smoke_test_docker_image() {
         return 1
     fi
     
-    # Run the container briefly to test if it starts correctly
+    # Sanitize the app name for container name
-    local container_name="smoke-test-${app_name//[^a-zA-Z0-9]/-}-$(date +%s)"
+    local clean_app_name=$(printf '%s\n' "$app_name" | sed 's/[^a-zA-Z0-9]/-/g' | tr -cd '[:alnum:]-')
+    local container_name="smoke-test-${clean_app_name:0:50}-$(date +%s)"
+    
+    # Validate container name doesn't exceed Docker limits
+    if [ ${#container_name} -gt 63 ]; then
+        container_name="${container_name:0:63}"
+    fi
     
     # Run without specific health check initially, just see if container starts and stays running
     if ! docker run -d --name "$container_name" "$docker_image" >/dev/null 2>&1; then
         echo "Failed to start container for $app_name during smoke test"
-        docker rm "$container_name" >/dev/null 2>&1 || true
+        # Remove container in case it was partially created
+        docker rm -f "$container_name" >/dev/null 2>&1 || true
         return 1
     fi
     
-    # Wait a few seconds to see if the container stays running
+    # Give the container time to start - wait with periodic checks
-    sleep 15
+    local max_wait=30  # Maximum wait time in seconds
+    local waited=0
+    local container_status="not_started"
     
-    # Check if the container is still running
+    while [ $waited -lt $max_wait ]; do
-    local container_status
+        container_status=$(docker inspect -f '{{.State.Status}}' "$container_name" 2>/dev/null || echo "not_found")
-    container_status=$(docker inspect -f '{{.State.Status}}' "$container_name" 2>/dev/null || echo "not_found")
+        if [ "$container_status" = "running" ]; then
+            break
+        elif [ "$container_status" = "exited" ] || [ "$container_status" = "dead" ]; then
+            # Container exited early, no need to wait longer
+            break
+        fi
+        sleep 2
+        waited=$((waited + 2))
+    done
     
     if [ "$container_status" = "running" ]; then
         echo "Smoke test passed for $app_name - container is running"
@@ -414,11 +559,11 @@ smoke_test_docker_image() {
         return 0
     else
         # Container stopped or crashed, get logs for debugging
-        echo "Container for $app_name did not stay running during smoke test (status: $container_status)"
+        echo "Container for $app_name did not stay running during smoke test (status: $container_status after ${waited}s)"
         echo "Container logs:"
-        docker logs "$container_name" 2>&1 | head -30
+        docker logs "$container_name" 2>/dev/null | head -30 || echo "Could not retrieve container logs"
-        docker stop "$container_name" >/dev/null 2>&1 || true
+        # Force remove the container
-        docker rm "$container_name" >/dev/null 2>&1 || true
+        docker rm -f "$container_name" >/dev/null 2>&1 || true
         return 1
     fi
 }
@@ -461,6 +606,38 @@ package_generic_app() {
     
     cd "$app_dir"
     
+    # Create .dockerignore to exclude sensitive files
+    cat > .dockerignore << 'DOCKERIGNORE_EOF'
+.git
+.gitignore
+*.env
+*.key
+*.pem
+*.crt
+*.cert
+Dockerfile
+.dockerignore
+*.log
+node_modules
+__pycache__
+.pytest_cache
+.coverage
+.vscode
+.idea
+*.swp
+*.swo
+.DS_Store
+Thumbs.db
+README.md
+CHANGELOG.md
+LICENSE
+AUTHORS
+CONTRIBUTORS
+config/
+secrets/
+tokens/
+DOCKERIGNORE_EOF
+    
     # Extract username/repo from the app_url for manifest
     local repo_path
     if [[ "$app_url" == *"github.com"* ]]; then
@@ -565,8 +742,8 @@ EXPOSE 8080
 CMD ["/run-app.sh"]
 DOCKERFILE_EOF
     
-    # Build Docker image
+    # Build Docker image with a more unique name to avoid conflicts in parallel execution
-    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest"
+    local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest"
     if ! docker build -t "$docker_image" .; then
         echo "Failed to build Docker image for $app_name"
         return 1
@@ -579,6 +756,6 @@ DOCKERFILE_EOF
     fi
     
     # Save the Docker image as an artifact
-    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz"
+    docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz"
     return 0
 }

CloudronStack/test_add_url.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Test script to verify the add_git_url functionality
+
+# Source the master script to get access to its functions
+source /home/localuser/TSYSDevStack/CloudronStack/output/master-control-script.sh
+
+# Test adding a new URL
+echo "Testing add_git_url function..."
+add_git_url "https://github.com/testuser/testrepo"
+
+# Check the git URL list file to see if the URL was added
+echo "Contents of GitUrlList.txt after adding:"
+cat /home/localuser/TSYSDevStack/CloudronStack/collab/GitUrlList.txt
+
+# Test adding the same URL again (should not duplicate)
+echo "Testing adding the same URL again (should not duplicate)..."
+add_git_url "https://github.com/testuser/testrepo"
+
+# Add another URL for good measure
+echo "Testing adding a second URL..."
+add_git_url "https://github.com/anotheruser/anotherrepo"
+
+echo "Test completed successfully!"