#!/bin/bash # Master Control Script for Cloudron Packaging # This script orchestrates the packaging of all applications from GitUrlList.txt # It runs three packaging projects in parallel and maintains status tracking set -e # Exit on any error set -u # Exit on undefined variables set -o pipefail # Exit on pipe failures # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OUTPUT_DIR="$SCRIPT_DIR/output" ARTIFACTS_DIR="$OUTPUT_DIR/CloudronPackages-Artifacts" WORKSPACES_DIR="$OUTPUT_DIR/CloudronPackages-Workspaces" STATUS_FILE="$SCRIPT_DIR/collab/STATUS.md" GIT_URL_LIST="$SCRIPT_DIR/collab/GitUrlList.txt" HUMAN_HELP_DIR="$WORKSPACES_DIR/human-help-required" MAX_RETRIES=5 LOG_FILE="$WORKSPACES_DIR/packaging.log" # Docker image prefix DOCKER_PREFIX="tsysdevstack-cloudron-buildtest-" # Source the packaging functions source "$SCRIPT_DIR/package-functions.sh" # Create necessary directories mkdir -p "$ARTIFACTS_DIR" "$WORKSPACES_DIR" "$HUMAN_HELP_DIR" # Function to log messages log_message() { local level=$1 local message=$2 local timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo "[$timestamp] [$level] $message" >> "$LOG_FILE" } # Function to perform audit of the packaging process perform_audit() { log_message "INFO" "Starting audit process" # Count total, completed, failed, and in-progress applications local total_count=$(grep -c "https://" "$GIT_URL_LIST" || echo 0) local completed_count=$(grep -c "✅ COMPLETE" "$STATUS_FILE" || echo 0) local failed_count=$(grep -c "🛑 FAILED" "$STATUS_FILE" || echo 0) local in_progress_count=$(grep -c "🔄 IN PROGRESS" "$STATUS_FILE" || echo 0) local pending_count=$((total_count - completed_count - failed_count - in_progress_count)) log_message "INFO" "Audit Summary - Total: $total_count, Completed: $completed_count, Failed: $failed_count, In Progress: $in_progress_count, Pending: $pending_count" # Check for artifacts directory health local artifact_count=$(find "$ARTIFACTS_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l) log_message "INFO" "Found $artifact_count artifact directories in $ARTIFACTS_DIR" # Check for workspace directory health local workspace_count=$(find "$WORKSPACES_DIR" -mindepth 1 -maxdepth 1 -type d | grep -v "human-help-required\|packaging.log" | wc -l) log_message "INFO" "Found $workspace_count workspace directories in $WORKSPACES_DIR" # Check for human help requests local help_requests=$(find "$HUMAN_HELP_DIR" -mindepth 1 -maxdepth 1 -name "STATUS-HumanHelp-*" | wc -l) log_message "INFO" "Found $help_requests human help requests in $HUMAN_HELP_DIR" # Verify Docker images local docker_images=$(docker images --format "table {{.Repository}}:{{.Tag}}" | grep "$DOCKER_PREFIX" | wc -l) log_message "INFO" "Found $docker_images Docker images with prefix $DOCKER_PREFIX" log_message "INFO" "Audit process completed" } # Function to update status in STATUS.md update_status() { local app_name=$1 local new_status=$2 local notes=${3:-""} # Escape special characters for sed local escaped_app_name=$(printf '%s\n' "$app_name" | sed 's/[[\.*^$()+?{|]/\\&/g') local escaped_status=$(printf '%s\n' "$new_status" | sed 's/[[\.*^$()+?{|]/\\&/g') local escaped_notes=$(printf '%s\n' "$notes" | sed 's/[[\.*^$()+?{|]/\\&/g' | sed 's/&/&/g; s//>/g') # Update status in the file - find the line with the app name and update its status sed -i "s/^| $escaped_app_name |.*|.*|.*$/| $app_name |.*| $new_status | $escaped_notes |/" "$STATUS_FILE" log_message "INFO" "Updated status for $app_name to $new_status" } # Function to get the repository name from URL get_repo_name() { local url=$1 if [[ -z "$url" ]]; then log_message "ERROR" "URL is empty in get_repo_name function" return 1 fi local repo_part repo_part=$(basename "$url") repo_part=${repo_part%.git} # Sanitize the repo name to contain only valid characters echo "$repo_part" | sed 's/[^a-zA-Z0-9._-]/-/g' } # Function to extract username/repo from URL for GitHub get_username_repo() { local url=$1 if [[ -z "$url" ]]; then log_message "ERROR" "URL is empty in get_username_repo function" return 1 fi if [[ "$url" == *"github.com"* ]]; then # Extract username/repo from GitHub URL local path=${url#*github.com/} path=${path%.git} echo "$path" elif [[ "$url" == *"gitlab.com"* ]]; then # Extract username/repo from GitLab URL local path=${url#*gitlab.com/} path=${path%.git} echo "$path" else # For other URLs, just return the repo name get_repo_name "$url" fi } # Function to run individual packaging script run_packaging_script() { local url=$1 local repo_name=$(get_repo_name "$url") local username_repo=$(get_username_repo "$url") local workspace_dir="$WORKSPACES_DIR/$repo_name" local artifact_dir="$ARTIFACTS_DIR/$repo_name" echo "$(date): Starting packaging for $repo_name ($url)" >> "$WORKSPACES_DIR/packaging.log" # Update status to IN PROGRESS update_status "$repo_name" "🔄 IN PROGRESS" "Packaging started" # Initialize workspace mkdir -p "$workspace_dir" "$artifact_dir" # Clone repository if [ ! -d "$workspace_dir/repo" ] || [ -z "$(ls -A "$workspace_dir/repo")" ]; then echo "Cloning $url to $workspace_dir/repo" if ! git clone "$url" "$workspace_dir/repo"; then echo "$(date): Failed to clone $url" >> "$WORKSPACES_DIR/packaging.log" update_status "$repo_name" "🛑 FAILED" "Failed to clone repository" return 1 fi else # Update repository echo "Updating $url in $workspace_dir/repo" if ! (cd "$workspace_dir/repo" && git fetch && git reset --hard origin/main 2>/dev/null || git reset --hard origin/master 2>/dev/null || git pull); then echo "$(date): Failed to update $url" >> "$WORKSPACES_DIR/packaging.log" update_status "$repo_name" "🔄 IN PROGRESS" "Repo update failed, will retry with fresh clone" # Remove the repo and try to clone again rm -rf "$workspace_dir/repo" if ! git clone "$url" "$workspace_dir/repo"; then echo "$(date): Failed to re-clone $url after update failure" >> "$WORKSPACES_DIR/packaging.log" update_status "$repo_name" "🛑 FAILED" "Failed to update or re-clone repository" return 1 fi fi fi # Attempt packaging with retries local attempt=1 local success=0 while [ $attempt -le $MAX_RETRIES ] && [ $success -eq 0 ]; do echo "$(date): Attempt $attempt/$MAX_RETRIES for $repo_name" >> "$WORKSPACES_DIR/packaging.log" # Capture the output and error of the packaging function if package_application "$repo_name" "$username_repo" "$workspace_dir" "$artifact_dir" 2>"$workspace_dir/error.log"; then success=1 update_status "$repo_name" "✅ COMPLETE" "Packaged successfully on attempt $attempt" echo "$(date): Successfully packaged $repo_name on attempt $attempt" >> "$WORKSPACES_DIR/packaging.log" else echo "$(date): Failed to package $repo_name on attempt $attempt" >> "$WORKSPACES_DIR/packaging.log" cat "$workspace_dir/error.log" >> "$WORKSPACES_DIR/packaging.log" ((attempt++)) fi done if [ $success -eq 0 ]; then # Mark as failed and create human help request with more detailed information local error_details=$(cat "$workspace_dir/error.log" | head -20 | sed 's/"/\\"/g' | tr '\n' ' ') update_status "$repo_name" "🛑 FAILED" "Failed after $MAX_RETRIES attempts. Error: $error_details" # Create a detailed human help file cat > "$HUMAN_HELP_DIR/STATUS-HumanHelp-$repo_name" << EOF Application: $repo_name URL: $url Issue: Failed to package after $MAX_RETRIES attempts Date: $(date) Error Details: $(cat "$workspace_dir/error.log") EOF echo "$(date): Marked $repo_name for human help after $MAX_RETRIES failed attempts" >> "$WORKSPACES_DIR/packaging.log" fi } # Function to package a specific application package_application() { local repo_name=$1 local username_repo=$2 local workspace_dir=$3 local artifact_dir=$4 local repo_path="$workspace_dir/repo" # Use the function library to detect and package the application detect_and_package "$repo_name" "$repo_path" "$artifact_dir" } # Function to create a Dockerfile based on the application type create_dockerfile() { local repo_name=$1 local repo_path=$2 # Detect application type and create appropriate Dockerfile # This is a simplified approach - in reality, this would be much more complex if [ -f "$repo_path/package.json" ]; then # Node.js application cat > "$repo_path/Dockerfile" << EOF FROM node:18-alpine WORKDIR /app COPY package*.json ./ RUN npm install COPY . . EXPOSE 3000 CMD ["npm", "start"] EOF elif [ -f "$repo_path/requirements.txt" ]; then # Python application cat > "$repo_path/Dockerfile" << EOF FROM python:3.11-slim WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . EXPOSE 8000 CMD ["python", "app.py"] EOF elif [ -f "$repo_path/composer.json" ]; then # PHP application cat > "$repo_path/Dockerfile" << EOF FROM php:8.1-apache RUN docker-php-ext-install mysqli && docker-php-ext-enable mysqli COPY . /var/www/html/ EXPOSE 80 CMD ["apache2-foreground"] EOF elif [ -f "$repo_path/Gemfile" ]; then # Ruby application cat > "$repo_path/Dockerfile" << EOF FROM ruby:3.0 WORKDIR /app COPY Gemfile Gemfile.lock ./ RUN bundle install COPY . . EXPOSE 3000 CMD ["ruby", "app.rb"] EOF else # Default to a basic server cat > "$repo_path/Dockerfile" << EOF FROM alpine:latest WORKDIR /app COPY . . RUN apk add --no-cache bash EXPOSE 8080 CMD ["sh", "-c", "while true; do sleep 30; done"] EOF fi } # Main function to process all applications main() { log_message "INFO" "Starting Cloudron packaging process" # Read URLs from GitUrlList.txt local urls=() while IFS= read -r line; do if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then urls+=("$line") fi done < "$GIT_URL_LIST" local total=${#urls[@]} log_message "INFO" "Found $total URLs to process" # Process applications in batches of 3 for parallel execution local i=0 while [ $i -lt $total ]; do # Process up to 3 applications in parallel local end=$((i + 3)) [ $end -gt $total ] && end=$total log_message "INFO" "Starting batch with applications $(printf '%s; ' "${urls[@]:i:3}")" for ((j = i; j < end; j++)); do log_message "INFO" "Starting packaging for ${urls[$j]}" run_packaging_script "${urls[$j]}" & done # Wait for all background processes to complete wait # Perform audit after each batch perform_audit # Update i for the next batch i=$end # Update progress summary in STATUS.md local completed=$(grep -o "✅ COMPLETE" "$STATUS_FILE" | wc -l) local failed=$(grep -o "🛑 FAILED" "$STATUS_FILE" | wc -l) local in_progress=$(grep -o "🔄 IN PROGRESS" "$STATUS_FILE" | wc -l) local pending=$((total - completed - failed - in_progress)) # Update summary section in STATUS.md sed -i '/## Progress Summary/Q' "$STATUS_FILE" cat >> "$STATUS_FILE" << EOF ## Progress Summary - Total Applications: $total - Completed: $completed ($(awk "BEGIN {printf \"%.0f\", $completed * 100 / $total}")%) - In Progress: $in_progress ($(awk "BEGIN {printf \"%.0f\", $in_progress * 100 / $total}")%) - Failed: $failed ($(awk "BEGIN {printf \"%.0f\", $failed * 100 / $total}")%) - Pending: $pending ($(awk "BEGIN {printf \"%.0f\", $pending * 100 / $total}")%) ## Human Help Required $(ls -1 "$HUMAN_HELP_DIR" 2>/dev/null || echo "None at the moment.") ## Last Updated $(date) EOF done # Final audit perform_audit log_message "INFO" "Completed Cloudron packaging process" } # Run the main function if script is executed directly if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main "$@" fi