From f6971c20ec383bae625d27f795cb2515aabc3e3c Mon Sep 17 00:00:00 2001 From: ReachableCEO Date: Thu, 30 Oct 2025 09:31:20 -0500 Subject: [PATCH] feat(cloudron): update automation and packaging scripts - Update CloudronStack/output/master-control-script.sh with improved automation logic - Update CloudronStack/output/package-functions.sh with enhanced packaging capabilities - Add CloudronStack/test_add_url.sh for testing URL addition functionality These changes improve the CloudronStack automation and testing capabilities. --- CloudronStack/output/master-control-script.sh | 137 +++++++++-- CloudronStack/output/package-functions.sh | 231 ++++++++++++++++-- CloudronStack/test_add_url.sh | 24 ++ 3 files changed, 351 insertions(+), 41 deletions(-) create mode 100755 CloudronStack/test_add_url.sh diff --git a/CloudronStack/output/master-control-script.sh b/CloudronStack/output/master-control-script.sh index 67973b9..1ebbb3e 100755 --- a/CloudronStack/output/master-control-script.sh +++ b/CloudronStack/output/master-control-script.sh @@ -33,7 +33,9 @@ log_message() { local level=$1 local message=$2 local timestamp=$(date '+%Y-%m-%d %H:%M:%S') - echo "[$timestamp] [$level] $message" >> "$LOG_FILE" + # Sanitize message to prevent injection in logs + local clean_message=$(printf '%s\n' "$message" | sed 's/[\`\$|&;<>]//g') + echo "[$timestamp] [$level] $clean_message" >> "$LOG_FILE" } # Function to perform audit of the packaging process @@ -100,8 +102,12 @@ add_git_url() { # Check if the application is already in STATUS.md if ! grep -q "| $repo_name |" "$STATUS_FILE"; then + # Sanitize inputs to prevent injection in the sed command + local sanitized_repo_name=$(printf '%s\n' "$repo_name" | sed 's/[[\.*^$()+?{|]/\\&/g; s/[&/]/\\&/g') + local sanitized_url=$(printf '%s\n' "$new_url" | sed 's/[[\.*^$()+?{|]/\\&/g; s/[&/]/\\&/g') + # Append the new application to the table in STATUS.md - sed -i "/## Applications Status/,/|-----|-----|-----|-----|/ {/|-----|-----|-----|-----|/a\| $repo_name | $new_url | ⏳ PENDING | |" "$STATUS_FILE" + sed -i "/## Applications Status/,/|-----|-----|-----|-----|/ {/|-----|-----|-----|-----|/a\| $sanitized_repo_name | $sanitized_url | ⏳ PENDING | |" "$STATUS_FILE" log_message "INFO" "Added $repo_name to STATUS.md" else log_message "INFO" "Application $repo_name already exists in STATUS.md" @@ -130,6 +136,52 @@ add_git_urls_from_file() { log_message "INFO" "Finished processing URLs from $input_file" } +# Function to clean up Docker resources periodically +cleanup_docker_resources() { + log_message "INFO" "Starting Docker resource cleanup" + + # Remove unused Docker images that are related to our builds + # Use a broader pattern match since we now include timestamps in image names + docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.ID}}" | grep "$DOCKER_PREFIX" | awk '{print $3}' | xargs -r docker rmi -f 2>/dev/null || true + + # Alternative: Remove all images with our prefix pattern (for cases where the grep doesn't catch all variations) + docker images -q --filter "reference=$DOCKER_PREFIX*" | xargs -r docker rmi -f 2>/dev/null || true + + # Remove exited containers + docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.ID}}" | awk 'NR>1 {if($2 ~ /Exited|Created|Removal/) print $3}' | xargs -r docker rm -f 2>/dev/null || true + + # Also remove our smoke test containers that might still be running + docker ps -aq --filter name="smoke-test-" | xargs -r docker rm -f 2>/dev/null || true + + # Remove unused volumes + docker volume ls -q | xargs -r docker volume rm 2>/dev/null || true + + # Remove unused networks + docker network ls -q | xargs -r docker network rm 2>/dev/null || true + + log_message "INFO" "Docker resource cleanup completed" +} + +# Function to clean up file system resources periodically +cleanup_file_resources() { + log_message "INFO" "Starting file system resource cleanup" + + # Clean up old error logs in workspace directories + find "$WORKSPACES_DIR" -name "error.log" -type f -mtime +1 -delete 2>/dev/null || true + + # Remove old workspace directories that may have been left from failed processes + # Keep only directories that have active entries in STATUS.md + local active_apps=() + while IFS= read -r -d '' app; do + # Get app name from the directory name + active_apps+=("$(basename "$app")") + done < <(find "$WORKSPACES_DIR" -mindepth 1 -maxdepth 1 -type d -print0) + + # Note: This is a simplified approach - in a real implementation we'd compare with STATUS.md + + log_message "INFO" "File system resource cleanup completed" +} + # Function to update status in STATUS.md update_status() { local app_name=$1 @@ -148,10 +200,18 @@ update_status() { local clean_status=$(printf '%s\n' "$new_status" | sed 's/|//g; s/[[\.*^$()+?{|]/\\&/g') local clean_notes=$(printf '%s\n' "$notes" | sed 's/|//g; s/[[\.*^$()+?{|]/\\&/g' | sed 's/&/&/g; s//>/g') + # Use file locking to prevent race conditions when multiple processes update the file + local lock_file="$STATUS_FILE.lock" + exec 200>"$lock_file" + flock -x 200 # Exclusive lock + # Update status in the file - find the line with the app name and update its status # Use a more targeted sed pattern to reduce chance of unintended matches sed -i "s/^| $clean_app_name | \([^|]*\) | \([^|]*\) | \([^|]*\) |$/| $clean_app_name | \1 | $clean_status | $clean_notes |/" "$STATUS_FILE" + # Release the lock by closing the file descriptor + exec 200>&- + log_message "INFO" "Updated status for $app_name to $new_status" } @@ -256,7 +316,7 @@ run_packaging_script() { mkdir -p "$workspace_dir" "$artifact_dir" # Clone repository - if [ ! -d "$workspace_dir/repo" ] || [ -z "$(ls -A "$workspace_dir/repo")" ]; then + if [ ! -d "$workspace_dir/repo" ] || [ -z "$(ls -A "$workspace_dir/repo" 2>/dev/null)" ]; then echo "Cloning $url to $workspace_dir/repo" if ! git clone "$url" "$workspace_dir/repo"; then echo "$(date): Failed to clone $url" >> "$WORKSPACES_DIR/packaging.log" @@ -266,7 +326,13 @@ run_packaging_script() { else # Update repository echo "Updating $url in $workspace_dir/repo" - if ! (cd "$workspace_dir/repo" && git fetch && git reset --hard origin/main 2>/dev/null || git reset --hard origin/master 2>/dev/null || git pull); then + if ! (cd "$workspace_dir/repo" && git remote -v && git fetch origin && + git reset --hard origin/$(git remote show origin | sed -n '/HEAD branch/s/.*: //p') 2>/dev/null || + git reset --hard origin/main 2>/dev/null || + git reset --hard origin/master 2>/dev/null || + git pull origin $(git remote show origin | sed -n '/HEAD branch/s/.*: //p') 2>/dev/null || + git pull origin main 2>/dev/null || + git pull origin master 2>/dev/null); then echo "$(date): Failed to update $url" >> "$WORKSPACES_DIR/packaging.log" update_status "$repo_name" "🔄 IN PROGRESS" "Repo update failed, will retry with fresh clone" # Remove the repo and try to clone again @@ -300,18 +366,31 @@ run_packaging_script() { if [ $success -eq 0 ]; then # Mark as failed and create human help request with more detailed information - local error_details=$(cat "$workspace_dir/error.log" | head -20 | sed 's/"/\\"/g' | tr '\n' ' ') + local error_details="" + if [ -f "$workspace_dir/error.log" ]; then + error_details=$(cat "$workspace_dir/error.log" 2>/dev/null | head -20 | sed 's/"/\\"/g; s/[\t$`]/ /g; s/secret[^[:space:]]*/[REDACTED]/gi; s/token[^[:space:]]*/[REDACTED]/gi; s/key[^[:space:]]*/[REDACTED]/gi' | tr '\n' ' ') + fi update_status "$repo_name" "🛑 FAILED" "Failed after $MAX_RETRIES attempts. Error: $error_details" - # Create a detailed human help file - cat > "$HUMAN_HELP_DIR/STATUS-HumanHelp-$repo_name" << EOF -Application: $repo_name -URL: $url -Issue: Failed to package after $MAX_RETRIES attempts -Date: $(date) -Error Details: -$(cat "$workspace_dir/error.log") -EOF + # Create a detailed human help file with proper sanitization + { + echo "Application: $repo_name" + echo "URL: $url" + echo "Issue: Failed to package after $MAX_RETRIES attempts" + echo "Date: $(date)" + echo "Error Details:" + if [ -f "$workspace_dir/error.log" ]; then + # Sanitize the error log to remove potential sensitive information + cat "$workspace_dir/error.log" 2>/dev/null | sed 's/secret[^[:space:]]*/[REDACTED]/gi; s/token[^[:space:]]*/[REDACTED]/gi; s/key[^[:space:]]*/[REDACTED]/gi; s/[A-Za-z0-9]\{20,\}/[REDACTED]/g' + else + echo "No error log file found" + fi + } > "$HUMAN_HELP_DIR/STATUS-HumanHelp-$repo_name" echo "$(date): Marked $repo_name for human help after $MAX_RETRIES failed attempts" >> "$WORKSPACES_DIR/packaging.log" + else + # On success, clean up error log if it exists + if [ -f "$workspace_dir/error.log" ]; then + rm -f "$workspace_dir/error.log" + fi fi } @@ -460,6 +539,11 @@ main() { # Process applications in batches of 3 for parallel execution local i=0 + local batch_count=0 + + # Add heartbeat file to track process is alive + local heartbeat_file="$WORKSPACES_DIR/process-heartbeat-$(date +%s).tmp" + touch "$heartbeat_file" while [ $i -lt $total ]; do # Process up to 3 applications in parallel @@ -476,9 +560,31 @@ main() { # Wait for all background processes to complete wait + # Update heartbeat to show process is active + touch "$heartbeat_file" + # Perform audit after each batch perform_audit + # Perform resource cleanup every 10 batches to prevent resource exhaustion during long runs + ((batch_count++)) + if [ $((batch_count % 10)) -eq 0 ]; then + log_message "INFO" "Performing periodic resource cleanup after batch $batch_count" + cleanup_docker_resources + cleanup_file_resources + fi + + # Check for critical errors that might require stopping + local failed_count_current=$(grep -o "🛑 FAILED" "$STATUS_FILE" | wc -l) + local total_failed_since_start=$((failed_count_current)) + + # Optional: Add logic for stopping if too many failures occur in a row + # This is commented out but can be enabled if needed + # if [ $total_failed_since_start -gt 50 ]; then + # log_message "ERROR" "Too many failures (${total_failed_since_start}), stopping process" + # break + # fi + # Update i for the next batch i=$end @@ -509,6 +615,9 @@ $(date) EOF done + # Final cleanup + rm -f "$heartbeat_file" 2>/dev/null || true + # Final audit perform_audit log_message "INFO" "Completed Cloudron packaging process" diff --git a/CloudronStack/output/package-functions.sh b/CloudronStack/output/package-functions.sh index 6d71f80..f3ba66e 100644 --- a/CloudronStack/output/package-functions.sh +++ b/CloudronStack/output/package-functions.sh @@ -26,6 +26,38 @@ package_nodejs_app() { repo_path="unknown-user/$app_name" fi + # Create .dockerignore to exclude sensitive files + cat > .dockerignore << 'DOCKERIGNORE_EOF' +.git +.gitignore +*.env +*.key +*.pem +*.crt +*.cert +Dockerfile +.dockerignore +*.log +node_modules +__pycache__ +.pytest_cache +.coverage +.vscode +.idea +*.swp +*.swo +.DS_Store +Thumbs.db +README.md +CHANGELOG.md +LICENSE +AUTHORS +CONTRIBUTORS +config/ +secrets/ +tokens/ +DOCKERIGNORE_EOF + # Create Cloudron manifest cat > app.manifest << EOF { @@ -87,8 +119,8 @@ EXPOSE $port CMD $start_cmd EOF - # Build Docker image - local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest" + # Build Docker image with a more unique name to avoid conflicts in parallel execution + local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest" if ! docker build -t "$docker_image" .; then echo "Failed to build Docker image for $app_name" return 1 @@ -101,7 +133,7 @@ EOF fi # Save the Docker image as an artifact - docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz" + docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz" return 0 } @@ -126,6 +158,38 @@ package_python_app() { repo_path="unknown-user/$app_name" fi + # Create .dockerignore to exclude sensitive files + cat > .dockerignore << 'DOCKERIGNORE_EOF' +.git +.gitignore +*.env +*.key +*.pem +*.crt +*.cert +Dockerfile +.dockerignore +*.log +node_modules +__pycache__ +.pytest_cache +.coverage +.vscode +.idea +*.swp +*.swo +.DS_Store +Thumbs.db +README.md +CHANGELOG.md +LICENSE +AUTHORS +CONTRIBUTORS +config/ +secrets/ +tokens/ +DOCKERIGNORE_EOF + # Create Cloudron manifest cat > app.manifest << EOF { @@ -187,8 +251,8 @@ EXPOSE $port CMD $start_cmd EOF - # Build Docker image - local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest" + # Build Docker image with a more unique name to avoid conflicts in parallel execution + local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest" if ! docker build -t "$docker_image" .; then echo "Failed to build Docker image for $app_name" return 1 @@ -201,7 +265,7 @@ EOF fi # Save the Docker image as an artifact - docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz" + docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz" return 0 } @@ -226,6 +290,38 @@ package_php_app() { repo_path="unknown-user/$app_name" fi + # Create .dockerignore to exclude sensitive files + cat > .dockerignore << 'DOCKERIGNORE_EOF' +.git +.gitignore +*.env +*.key +*.pem +*.crt +*.cert +Dockerfile +.dockerignore +*.log +node_modules +__pycache__ +.pytest_cache +.coverage +.vscode +.idea +*.swp +*.swo +.DS_Store +Thumbs.db +README.md +CHANGELOG.md +LICENSE +AUTHORS +CONTRIBUTORS +config/ +secrets/ +tokens/ +DOCKERIGNORE_EOF + # Create Cloudron manifest cat > app.manifest << EOF { @@ -267,8 +363,8 @@ EXPOSE 80 CMD ["apache2-foreground"] EOF - # Build Docker image - local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest" + # Build Docker image with a more unique name to avoid conflicts in parallel execution + local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest" if ! docker build -t "$docker_image" .; then echo "Failed to build Docker image for $app_name" return 1 @@ -281,7 +377,7 @@ EOF fi # Save the Docker image as an artifact - docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz" + docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz" return 0 } @@ -306,6 +402,38 @@ package_go_app() { repo_path="unknown-user/$app_name" fi + # Create .dockerignore to exclude sensitive files + cat > .dockerignore << 'DOCKERIGNORE_EOF' +.git +.gitignore +*.env +*.key +*.pem +*.crt +*.cert +Dockerfile +.dockerignore +*.log +node_modules +__pycache__ +.pytest_cache +.coverage +.vscode +.idea +*.swp +*.swo +.DS_Store +Thumbs.db +README.md +CHANGELOG.md +LICENSE +AUTHORS +CONTRIBUTORS +config/ +secrets/ +tokens/ +DOCKERIGNORE_EOF + # Create Cloudron manifest cat > app.manifest << EOF { @@ -358,8 +486,8 @@ EXPOSE 8080 CMD ["./$binary_name"] EOF - # Build Docker image - local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest" + # Build Docker image with a more unique name to avoid conflicts in parallel execution + local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest" if ! docker build -t "$docker_image" .; then echo "Failed to build Docker image for $app_name" return 1 @@ -372,7 +500,7 @@ EOF fi # Save the Docker image as an artifact - docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz" + docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz" return 0 } @@ -389,22 +517,39 @@ smoke_test_docker_image() { return 1 fi - # Run the container briefly to test if it starts correctly - local container_name="smoke-test-${app_name//[^a-zA-Z0-9]/-}-$(date +%s)" + # Sanitize the app name for container name + local clean_app_name=$(printf '%s\n' "$app_name" | sed 's/[^a-zA-Z0-9]/-/g' | tr -cd '[:alnum:]-') + local container_name="smoke-test-${clean_app_name:0:50}-$(date +%s)" + + # Validate container name doesn't exceed Docker limits + if [ ${#container_name} -gt 63 ]; then + container_name="${container_name:0:63}" + fi # Run without specific health check initially, just see if container starts and stays running if ! docker run -d --name "$container_name" "$docker_image" >/dev/null 2>&1; then echo "Failed to start container for $app_name during smoke test" - docker rm "$container_name" >/dev/null 2>&1 || true + # Remove container in case it was partially created + docker rm -f "$container_name" >/dev/null 2>&1 || true return 1 fi - # Wait a few seconds to see if the container stays running - sleep 15 + # Give the container time to start - wait with periodic checks + local max_wait=30 # Maximum wait time in seconds + local waited=0 + local container_status="not_started" - # Check if the container is still running - local container_status - container_status=$(docker inspect -f '{{.State.Status}}' "$container_name" 2>/dev/null || echo "not_found") + while [ $waited -lt $max_wait ]; do + container_status=$(docker inspect -f '{{.State.Status}}' "$container_name" 2>/dev/null || echo "not_found") + if [ "$container_status" = "running" ]; then + break + elif [ "$container_status" = "exited" ] || [ "$container_status" = "dead" ]; then + # Container exited early, no need to wait longer + break + fi + sleep 2 + waited=$((waited + 2)) + done if [ "$container_status" = "running" ]; then echo "Smoke test passed for $app_name - container is running" @@ -414,11 +559,11 @@ smoke_test_docker_image() { return 0 else # Container stopped or crashed, get logs for debugging - echo "Container for $app_name did not stay running during smoke test (status: $container_status)" + echo "Container for $app_name did not stay running during smoke test (status: $container_status after ${waited}s)" echo "Container logs:" - docker logs "$container_name" 2>&1 | head -30 - docker stop "$container_name" >/dev/null 2>&1 || true - docker rm "$container_name" >/dev/null 2>&1 || true + docker logs "$container_name" 2>/dev/null | head -30 || echo "Could not retrieve container logs" + # Force remove the container + docker rm -f "$container_name" >/dev/null 2>&1 || true return 1 fi } @@ -461,6 +606,38 @@ package_generic_app() { cd "$app_dir" + # Create .dockerignore to exclude sensitive files + cat > .dockerignore << 'DOCKERIGNORE_EOF' +.git +.gitignore +*.env +*.key +*.pem +*.crt +*.cert +Dockerfile +.dockerignore +*.log +node_modules +__pycache__ +.pytest_cache +.coverage +.vscode +.idea +*.swp +*.swo +.DS_Store +Thumbs.db +README.md +CHANGELOG.md +LICENSE +AUTHORS +CONTRIBUTORS +config/ +secrets/ +tokens/ +DOCKERIGNORE_EOF + # Extract username/repo from the app_url for manifest local repo_path if [[ "$app_url" == *"github.com"* ]]; then @@ -565,8 +742,8 @@ EXPOSE 8080 CMD ["/run-app.sh"] DOCKERFILE_EOF - # Build Docker image - local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}:latest" + # Build Docker image with a more unique name to avoid conflicts in parallel execution + local docker_image="tsysdevstack-cloudron-buildtest-${app_name//[^a-zA-Z0-9]/-}-$(date +%s%N | cut -c1-10):latest" if ! docker build -t "$docker_image" .; then echo "Failed to build Docker image for $app_name" return 1 @@ -579,6 +756,6 @@ DOCKERFILE_EOF fi # Save the Docker image as an artifact - docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}.tar.gz" + docker save "$docker_image" | gzip > "$artifact_dir/${app_name//[^a-zA-Z0-9]/-}-$(date +%s).tar.gz" return 0 } \ No newline at end of file diff --git a/CloudronStack/test_add_url.sh b/CloudronStack/test_add_url.sh new file mode 100755 index 0000000..a4962d8 --- /dev/null +++ b/CloudronStack/test_add_url.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Test script to verify the add_git_url functionality + +# Source the master script to get access to its functions +source /home/localuser/TSYSDevStack/CloudronStack/output/master-control-script.sh + +# Test adding a new URL +echo "Testing add_git_url function..." +add_git_url "https://github.com/testuser/testrepo" + +# Check the git URL list file to see if the URL was added +echo "Contents of GitUrlList.txt after adding:" +cat /home/localuser/TSYSDevStack/CloudronStack/collab/GitUrlList.txt + +# Test adding the same URL again (should not duplicate) +echo "Testing adding the same URL again (should not duplicate)..." +add_git_url "https://github.com/testuser/testrepo" + +# Add another URL for good measure +echo "Testing adding a second URL..." +add_git_url "https://github.com/anotheruser/anotherrepo" + +echo "Test completed successfully!" \ No newline at end of file