ReachableCEO
dd474374d4
- Update ToolboxStack/output/toolbox-base/Dockerfile with latest container configurations - Update ToolboxStack/output/toolbox-base/PROMPT with enhanced AI collaboration instructions - Update ToolboxStack/output/toolbox-base/README.md with current documentation - Add ToolboxStack/collab/tool-additions/ directory for additional tool configurations - Update CloudronStack/output/master-control-script.sh with improved automation logic These changes enhance the developer workspace configuration and improve automation workflows across the project.
355 lines
12 KiB
Bash
Executable File
355 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Master Control Script for Cloudron Packaging
|
|
# This script orchestrates the packaging of all applications from GitUrlList.txt
|
|
# It runs three packaging projects in parallel and maintains status tracking
|
|
|
|
set -e # Exit on any error
|
|
set -u # Exit on undefined variables
|
|
set -o pipefail # Exit on pipe failures
|
|
|
|
# Configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
OUTPUT_DIR="$SCRIPT_DIR/output"
|
|
ARTIFACTS_DIR="$OUTPUT_DIR/CloudronPackages-Artifacts"
|
|
WORKSPACES_DIR="$OUTPUT_DIR/CloudronPackages-Workspaces"
|
|
STATUS_FILE="$SCRIPT_DIR/collab/STATUS.md"
|
|
GIT_URL_LIST="$SCRIPT_DIR/collab/GitUrlList.txt"
|
|
HUMAN_HELP_DIR="$WORKSPACES_DIR/human-help-required"
|
|
MAX_RETRIES=5
|
|
LOG_FILE="$WORKSPACES_DIR/packaging.log"
|
|
|
|
# Docker image prefix
|
|
DOCKER_PREFIX="tsysdevstack-cloudron-buildtest-"
|
|
|
|
# Source the packaging functions
|
|
source "$SCRIPT_DIR/package-functions.sh"
|
|
|
|
# Create necessary directories
|
|
mkdir -p "$ARTIFACTS_DIR" "$WORKSPACES_DIR" "$HUMAN_HELP_DIR"
|
|
|
|
# Function to log messages
|
|
log_message() {
|
|
local level=$1
|
|
local message=$2
|
|
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo "[$timestamp] [$level] $message" >> "$LOG_FILE"
|
|
}
|
|
|
|
# Function to perform audit of the packaging process
|
|
perform_audit() {
|
|
log_message "INFO" "Starting audit process"
|
|
|
|
# Count total, completed, failed, and in-progress applications
|
|
local total_count=$(grep -c "https://" "$GIT_URL_LIST" || echo 0)
|
|
local completed_count=$(grep -c "✅ COMPLETE" "$STATUS_FILE" || echo 0)
|
|
local failed_count=$(grep -c "🛑 FAILED" "$STATUS_FILE" || echo 0)
|
|
local in_progress_count=$(grep -c "🔄 IN PROGRESS" "$STATUS_FILE" || echo 0)
|
|
local pending_count=$((total_count - completed_count - failed_count - in_progress_count))
|
|
|
|
log_message "INFO" "Audit Summary - Total: $total_count, Completed: $completed_count, Failed: $failed_count, In Progress: $in_progress_count, Pending: $pending_count"
|
|
|
|
# Check for artifacts directory health
|
|
local artifact_count=$(find "$ARTIFACTS_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l)
|
|
log_message "INFO" "Found $artifact_count artifact directories in $ARTIFACTS_DIR"
|
|
|
|
# Check for workspace directory health
|
|
local workspace_count=$(find "$WORKSPACES_DIR" -mindepth 1 -maxdepth 1 -type d | grep -v "human-help-required\|packaging.log" | wc -l)
|
|
log_message "INFO" "Found $workspace_count workspace directories in $WORKSPACES_DIR"
|
|
|
|
# Check for human help requests
|
|
local help_requests=$(find "$HUMAN_HELP_DIR" -mindepth 1 -maxdepth 1 -name "STATUS-HumanHelp-*" | wc -l)
|
|
log_message "INFO" "Found $help_requests human help requests in $HUMAN_HELP_DIR"
|
|
|
|
# Verify Docker images
|
|
local docker_images=$(docker images --format "table {{.Repository}}:{{.Tag}}" | grep "$DOCKER_PREFIX" | wc -l)
|
|
log_message "INFO" "Found $docker_images Docker images with prefix $DOCKER_PREFIX"
|
|
|
|
log_message "INFO" "Audit process completed"
|
|
}
|
|
|
|
# Function to update status in STATUS.md
|
|
update_status() {
|
|
local app_name=$1
|
|
local new_status=$2
|
|
local notes=${3:-""}
|
|
|
|
# Escape special characters for sed
|
|
local escaped_app_name=$(echo "$app_name" | sed 's/[[\.*^$()+?{|]/\\&/g')
|
|
local escaped_status=$(echo "$new_status" | sed 's/[[\.*^$()+?{|]/\\&/g')
|
|
local escaped_notes=$(echo "$notes" | sed 's/[[\.*^$()+?{|]/\\&/g' | sed 's/&/&/g; s/</</g; s/>/>/g')
|
|
|
|
# Update status in the file - find the line with the app name and update its status
|
|
sed -i "s/^| $escaped_app_name |.*|.*|.*$/| $app_name |.*| $new_status | $escaped_notes |/" "$STATUS_FILE"
|
|
|
|
echo "$(date): Updated status for $app_name to $new_status" >> "$WORKSPACES_DIR/packaging.log"
|
|
}
|
|
|
|
# Function to get the repository name from URL
|
|
get_repo_name() {
|
|
local url=$1
|
|
if [[ "$url" == *"github.com"* ]]; then
|
|
echo "${url##*/}" | sed 's/\.git$//'
|
|
elif [[ "$url" == *"gitlab.com"* ]]; then
|
|
echo "${url##*/}" | sed 's/\.git$//'
|
|
else
|
|
echo "${url##*/}" | sed 's/\.git$//'
|
|
fi
|
|
}
|
|
|
|
# Function to extract username/repo from URL for GitHub
|
|
get_username_repo() {
|
|
local url=$1
|
|
if [[ "$url" == *"github.com"* ]]; then
|
|
# Extract username/repo from GitHub URL
|
|
echo "${url#*github.com/}" | sed 's/\.git$//'
|
|
elif [[ "$url" == *"gitlab.com"* ]]; then
|
|
# Extract username/repo from GitLab URL
|
|
echo "${url#*gitlab.com/}" | sed 's/\.git$//'
|
|
else
|
|
# For other URLs, just return the repo name
|
|
echo "$(get_repo_name "$url")"
|
|
fi
|
|
}
|
|
|
|
# Function to run individual packaging script
|
|
run_packaging_script() {
|
|
local url=$1
|
|
local repo_name=$(get_repo_name "$url")
|
|
local username_repo=$(get_username_repo "$url")
|
|
local workspace_dir="$WORKSPACES_DIR/$repo_name"
|
|
local artifact_dir="$ARTIFACTS_DIR/$repo_name"
|
|
|
|
echo "$(date): Starting packaging for $repo_name ($url)" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
# Update status to IN PROGRESS
|
|
update_status "$repo_name" "🔄 IN PROGRESS" "Packaging started"
|
|
|
|
# Initialize workspace
|
|
mkdir -p "$workspace_dir" "$artifact_dir"
|
|
|
|
# Clone repository
|
|
if [ ! -d "$workspace_dir/repo" ] || [ -z "$(ls -A "$workspace_dir/repo")" ]; then
|
|
echo "Cloning $url to $workspace_dir/repo"
|
|
if ! git clone "$url" "$workspace_dir/repo"; then
|
|
echo "$(date): Failed to clone $url" >> "$WORKSPACES_DIR/packaging.log"
|
|
update_status "$repo_name" "🛑 FAILED" "Failed to clone repository"
|
|
return 1
|
|
fi
|
|
else
|
|
# Update repository
|
|
echo "Updating $url in $workspace_dir/repo"
|
|
if ! (cd "$workspace_dir/repo" && git fetch && git reset --hard origin/main 2>/dev/null || git reset --hard origin/master 2>/dev/null || git pull); then
|
|
echo "$(date): Failed to update $url" >> "$WORKSPACES_DIR/packaging.log"
|
|
update_status "$repo_name" "🔄 IN PROGRESS" "Repo update failed, will retry with fresh clone"
|
|
# Remove the repo and try to clone again
|
|
rm -rf "$workspace_dir/repo"
|
|
if ! git clone "$url" "$workspace_dir/repo"; then
|
|
echo "$(date): Failed to re-clone $url after update failure" >> "$WORKSPACES_DIR/packaging.log"
|
|
update_status "$repo_name" "🛑 FAILED" "Failed to update or re-clone repository"
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Attempt packaging with retries
|
|
local attempt=1
|
|
local success=0
|
|
|
|
while [ $attempt -le $MAX_RETRIES ] && [ $success -eq 0 ]; do
|
|
echo "$(date): Attempt $attempt/$MAX_RETRIES for $repo_name" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
# Capture the output and error of the packaging function
|
|
if package_application "$repo_name" "$username_repo" "$workspace_dir" "$artifact_dir" 2>"$workspace_dir/error.log"; then
|
|
success=1
|
|
update_status "$repo_name" "✅ COMPLETE" "Packaged successfully on attempt $attempt"
|
|
echo "$(date): Successfully packaged $repo_name on attempt $attempt" >> "$WORKSPACES_DIR/packaging.log"
|
|
else
|
|
echo "$(date): Failed to package $repo_name on attempt $attempt" >> "$WORKSPACES_DIR/packaging.log"
|
|
cat "$workspace_dir/error.log" >> "$WORKSPACES_DIR/packaging.log"
|
|
((attempt++))
|
|
fi
|
|
done
|
|
|
|
if [ $success -eq 0 ]; then
|
|
# Mark as failed and create human help request with more detailed information
|
|
local error_details=$(cat "$workspace_dir/error.log" | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')
|
|
update_status "$repo_name" "🛑 FAILED" "Failed after $MAX_RETRIES attempts. Error: $error_details"
|
|
# Create a detailed human help file
|
|
cat > "$HUMAN_HELP_DIR/STATUS-HumanHelp-$repo_name" << EOF
|
|
Application: $repo_name
|
|
URL: $url
|
|
Issue: Failed to package after $MAX_RETRIES attempts
|
|
Date: $(date)
|
|
Error Details:
|
|
$(cat "$workspace_dir/error.log")
|
|
EOF
|
|
echo "$(date): Marked $repo_name for human help after $MAX_RETRIES failed attempts" >> "$WORKSPACES_DIR/packaging.log"
|
|
fi
|
|
}
|
|
|
|
# Function to package a specific application
|
|
package_application() {
|
|
local repo_name=$1
|
|
local username_repo=$2
|
|
local workspace_dir=$3
|
|
local artifact_dir=$4
|
|
|
|
local repo_path="$workspace_dir/repo"
|
|
|
|
# Use the function library to detect and package the application
|
|
detect_and_package "$repo_name" "$repo_path" "$artifact_dir"
|
|
}
|
|
|
|
# Function to create a Dockerfile based on the application type
|
|
create_dockerfile() {
|
|
local repo_name=$1
|
|
local repo_path=$2
|
|
|
|
# Detect application type and create appropriate Dockerfile
|
|
# This is a simplified approach - in reality, this would be much more complex
|
|
|
|
if [ -f "$repo_path/package.json" ]; then
|
|
# Node.js application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM node:18-alpine
|
|
|
|
WORKDIR /app
|
|
|
|
COPY package*.json ./
|
|
RUN npm install
|
|
|
|
COPY . .
|
|
|
|
EXPOSE 3000
|
|
|
|
CMD ["npm", "start"]
|
|
EOF
|
|
elif [ -f "$repo_path/requirements.txt" ]; then
|
|
# Python application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM python:3.11-slim
|
|
|
|
WORKDIR /app
|
|
|
|
COPY requirements.txt .
|
|
RUN pip install --no-cache-dir -r requirements.txt
|
|
|
|
COPY . .
|
|
|
|
EXPOSE 8000
|
|
|
|
CMD ["python", "app.py"]
|
|
EOF
|
|
elif [ -f "$repo_path/composer.json" ]; then
|
|
# PHP application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM php:8.1-apache
|
|
|
|
RUN docker-php-ext-install mysqli && docker-php-ext-enable mysqli
|
|
|
|
COPY . /var/www/html/
|
|
|
|
EXPOSE 80
|
|
|
|
CMD ["apache2-foreground"]
|
|
EOF
|
|
elif [ -f "$repo_path/Gemfile" ]; then
|
|
# Ruby application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM ruby:3.0
|
|
|
|
WORKDIR /app
|
|
|
|
COPY Gemfile Gemfile.lock ./
|
|
RUN bundle install
|
|
|
|
COPY . .
|
|
|
|
EXPOSE 3000
|
|
|
|
CMD ["ruby", "app.rb"]
|
|
EOF
|
|
else
|
|
# Default to a basic server
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM alpine:latest
|
|
|
|
WORKDIR /app
|
|
|
|
COPY . .
|
|
|
|
RUN apk add --no-cache bash
|
|
|
|
EXPOSE 8080
|
|
|
|
CMD ["sh", "-c", "while true; do sleep 30; done"]
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# Main function to process all applications
|
|
main() {
|
|
echo "$(date): Starting Cloudron packaging process" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
# Read URLs from GitUrlList.txt
|
|
local urls=()
|
|
while IFS= read -r line; do
|
|
if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then
|
|
urls+=("$line")
|
|
fi
|
|
done < "$GIT_URL_LIST"
|
|
|
|
# Process applications in batches of 3 for parallel execution
|
|
local i=0
|
|
local total=${#urls[@]}
|
|
|
|
while [ $i -lt $total ]; do
|
|
# Process up to 3 applications in parallel
|
|
local end=$((i + 3))
|
|
[ $end -gt $total ] && end=$total
|
|
|
|
echo "$(date): Starting batch with applications $(printf '%s; ' "${urls[@]:i:3}")" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
for ((j = i; j < end; j++)); do
|
|
echo "$(date): Starting packaging for ${urls[$j]}" >> "$WORKSPACES_DIR/packaging.log"
|
|
run_packaging_script "${urls[$j]}" &
|
|
done
|
|
|
|
# Wait for all background processes to complete
|
|
wait
|
|
|
|
# Update i for the next batch
|
|
i=$end
|
|
|
|
# Update progress summary in STATUS.md
|
|
local completed=$(grep -o "✅ COMPLETE" "$STATUS_FILE" | wc -l)
|
|
local failed=$(grep -o "🛑 FAILED" "$STATUS_FILE" | wc -l)
|
|
local in_progress=$(grep -o "🔄 IN PROGRESS" "$STATUS_FILE" | wc -l)
|
|
local pending=$((total - completed - failed - in_progress))
|
|
|
|
# Update summary section in STATUS.md
|
|
sed -i '/## Progress Summary/Q' "$STATUS_FILE"
|
|
cat >> "$STATUS_FILE" << EOF
|
|
## Progress Summary
|
|
- Total Applications: $total
|
|
- Completed: $completed ($(awk "BEGIN {printf \"%.0f\", $completed * 100 / $total}")%)
|
|
- In Progress: $in_progress ($(awk "BEGIN {printf \"%.0f\", $in_progress * 100 / $total}")%)
|
|
- Failed: $failed ($(awk "BEGIN {printf \"%.0f\", $failed * 100 / $total}")%)
|
|
- Pending: $pending ($(awk "BEGIN {printf \"%.0f\", $pending * 100 / $total}")%)
|
|
|
|
## Human Help Required
|
|
$(ls -1 "$HUMAN_HELP_DIR" 2>/dev/null || echo "None at the moment.")
|
|
|
|
## Last Updated
|
|
$(date)
|
|
EOF
|
|
done
|
|
|
|
echo "$(date): Completed Cloudron packaging process" >> "$WORKSPACES_DIR/packaging.log"
|
|
}
|
|
|
|
# Run the main function if script is executed directly
|
|
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
main "$@"
|
|
fi |