ReachableCEO
77e10af05c
- Add new PROMPT file in collab directory for AI collaboration guidance - Add STATUS.md file in collab directory to track current status - Create output directory for project artifacts - Remove redundant commit-template.txt that is now centralized at top level - Update collab directory structure and content for better organization These changes improve the CloudronStack component's structure and documentation for better collaboration.
289 lines
8.9 KiB
Bash
Executable File
289 lines
8.9 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Master Control Script for Cloudron Packaging
|
|
# This script orchestrates the packaging of all applications from GitUrlList.txt
|
|
# It runs three packaging projects in parallel and maintains status tracking
|
|
|
|
set -e # Exit on any error
|
|
|
|
# Configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
OUTPUT_DIR="$SCRIPT_DIR/output"
|
|
ARTIFACTS_DIR="$OUTPUT_DIR/CloudronPackages-Artifacts"
|
|
WORKSPACES_DIR="$OUTPUT_DIR/CloudronPackages-Workspaces"
|
|
STATUS_FILE="$SCRIPT_DIR/collab/STATUS.md"
|
|
GIT_URL_LIST="$SCRIPT_DIR/collab/GitUrlList.txt"
|
|
HUMAN_HELP_DIR="$WORKSPACES_DIR/human-help-required"
|
|
MAX_RETRIES=5
|
|
|
|
# Docker image prefix
|
|
DOCKER_PREFIX="tsysdevstack-cloudron-buildtest-"
|
|
|
|
# Source the packaging functions
|
|
source "$SCRIPT_DIR/package-functions.sh"
|
|
|
|
# Create necessary directories
|
|
mkdir -p "$ARTIFACTS_DIR" "$WORKSPACES_DIR" "$HUMAN_HELP_DIR"
|
|
|
|
# Function to update status in STATUS.md
|
|
update_status() {
|
|
local app_name=$1
|
|
local new_status=$2
|
|
local notes=${3:-""}
|
|
|
|
# Escape special characters for sed
|
|
local escaped_app_name=$(echo "$app_name" | sed 's/[[\.*^$()+?{|]/\\&/g')
|
|
local escaped_status=$(echo "$new_status" | sed 's/[[\.*^$()+?{|]/\\&/g')
|
|
local escaped_notes=$(echo "$notes" | sed 's/[[\.*^$()+?{|]/\\&/g' | sed 's/&/&/g; s/</</g; s/>/>/g')
|
|
|
|
# Update status in the file - find the line with the app name and update its status
|
|
sed -i "s/^| $escaped_app_name |.*|.*|.*$/| $app_name |.*| $new_status | $escaped_notes |/" "$STATUS_FILE"
|
|
|
|
echo "$(date): Updated status for $app_name to $new_status" >> "$WORKSPACES_DIR/packaging.log"
|
|
}
|
|
|
|
# Function to get the repository name from URL
|
|
get_repo_name() {
|
|
local url=$1
|
|
if [[ "$url" == *"github.com"* ]]; then
|
|
echo "${url##*/}" | sed 's/\.git$//'
|
|
elif [[ "$url" == *"gitlab.com"* ]]; then
|
|
echo "${url##*/}" | sed 's/\.git$//'
|
|
else
|
|
echo "${url##*/}" | sed 's/\.git$//'
|
|
fi
|
|
}
|
|
|
|
# Function to extract username/repo from URL for GitHub
|
|
get_username_repo() {
|
|
local url=$1
|
|
if [[ "$url" == *"github.com"* ]]; then
|
|
# Extract username/repo from GitHub URL
|
|
echo "${url#*github.com/}" | sed 's/\.git$//'
|
|
elif [[ "$url" == *"gitlab.com"* ]]; then
|
|
# Extract username/repo from GitLab URL
|
|
echo "${url#*gitlab.com/}" | sed 's/\.git$//'
|
|
else
|
|
# For other URLs, just return the repo name
|
|
echo "$(get_repo_name "$url")"
|
|
fi
|
|
}
|
|
|
|
# Function to run individual packaging script
|
|
run_packaging_script() {
|
|
local url=$1
|
|
local repo_name=$(get_repo_name "$url")
|
|
local username_repo=$(get_username_repo "$url")
|
|
local workspace_dir="$WORKSPACES_DIR/$repo_name"
|
|
local artifact_dir="$ARTIFACTS_DIR/$repo_name"
|
|
local packaging_script="$WORKSPACES_DIR/packaging-$repo_name.sh"
|
|
|
|
echo "$(date): Starting packaging for $repo_name ($url)" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
# Update status to IN PROGRESS
|
|
update_status "$repo_name" "🔄 IN PROGRESS" "Packaging started"
|
|
|
|
# Initialize workspace
|
|
mkdir -p "$workspace_dir" "$artifact_dir"
|
|
|
|
# Clone repository
|
|
if [ ! -d "$workspace_dir/repo" ] || [ -z "$(ls -A "$workspace_dir/repo")" ]; then
|
|
echo "Cloning $url to $workspace_dir/repo"
|
|
git clone "$url" "$workspace_dir/repo"
|
|
else
|
|
# Update repository
|
|
echo "Updating $url in $workspace_dir/repo"
|
|
(cd "$workspace_dir/repo" && git fetch && git reset --hard origin/main 2>/dev/null || git reset --hard origin/master 2>/dev/null || git pull)
|
|
fi
|
|
|
|
# Attempt packaging with retries
|
|
local attempt=1
|
|
local success=0
|
|
|
|
while [ $attempt -le $MAX_RETRIES ] && [ $success -eq 0 ]; do
|
|
echo "$(date): Attempt $attempt/$MAX_RETRIES for $repo_name" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
# Run the packaging process (this would call the specific packaging function)
|
|
if package_application "$repo_name" "$username_repo" "$workspace_dir" "$artifact_dir"; then
|
|
success=1
|
|
update_status "$repo_name" "✅ COMPLETE" "Packaged successfully on attempt $attempt"
|
|
echo "$(date): Successfully packaged $repo_name on attempt $attempt" >> "$WORKSPACES_DIR/packaging.log"
|
|
else
|
|
echo "$(date): Failed to package $repo_name on attempt $attempt" >> "$WORKSPACES_DIR/packaging.log"
|
|
((attempt++))
|
|
fi
|
|
done
|
|
|
|
if [ $success -eq 0 ]; then
|
|
# Mark as failed and create human help request
|
|
update_status "$repo_name" "🛑 FAILED" "Failed after $MAX_RETRIES attempts"
|
|
touch "$HUMAN_HELP_DIR/STATUS-HumanHelp-$repo_name"
|
|
echo "$(date): Marked $repo_name for human help after $MAX_RETRIES failed attempts" >> "$WORKSPACES_DIR/packaging.log"
|
|
fi
|
|
}
|
|
|
|
# Function to package a specific application
|
|
package_application() {
|
|
local repo_name=$1
|
|
local username_repo=$2
|
|
local workspace_dir=$3
|
|
local artifact_dir=$4
|
|
|
|
local repo_path="$workspace_dir/repo"
|
|
|
|
# Use the function library to detect and package the application
|
|
detect_and_package "$repo_name" "$repo_path" "$artifact_dir"
|
|
}
|
|
|
|
# Function to create a Dockerfile based on the application type
|
|
create_dockerfile() {
|
|
local repo_name=$1
|
|
local repo_path=$2
|
|
|
|
# Detect application type and create appropriate Dockerfile
|
|
# This is a simplified approach - in reality, this would be much more complex
|
|
|
|
if [ -f "$repo_path/package.json" ]; then
|
|
# Node.js application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM node:18-alpine
|
|
|
|
WORKDIR /app
|
|
|
|
COPY package*.json ./
|
|
RUN npm install
|
|
|
|
COPY . .
|
|
|
|
EXPOSE 3000
|
|
|
|
CMD ["npm", "start"]
|
|
EOF
|
|
elif [ -f "$repo_path/requirements.txt" ]; then
|
|
# Python application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM python:3.11-slim
|
|
|
|
WORKDIR /app
|
|
|
|
COPY requirements.txt .
|
|
RUN pip install --no-cache-dir -r requirements.txt
|
|
|
|
COPY . .
|
|
|
|
EXPOSE 8000
|
|
|
|
CMD ["python", "app.py"]
|
|
EOF
|
|
elif [ -f "$repo_path/composer.json" ]; then
|
|
# PHP application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM php:8.1-apache
|
|
|
|
RUN docker-php-ext-install mysqli && docker-php-ext-enable mysqli
|
|
|
|
COPY . /var/www/html/
|
|
|
|
EXPOSE 80
|
|
|
|
CMD ["apache2-foreground"]
|
|
EOF
|
|
elif [ -f "$repo_path/Gemfile" ]; then
|
|
# Ruby application
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM ruby:3.0
|
|
|
|
WORKDIR /app
|
|
|
|
COPY Gemfile Gemfile.lock ./
|
|
RUN bundle install
|
|
|
|
COPY . .
|
|
|
|
EXPOSE 3000
|
|
|
|
CMD ["ruby", "app.rb"]
|
|
EOF
|
|
else
|
|
# Default to a basic server
|
|
cat > "$repo_path/Dockerfile" << EOF
|
|
FROM alpine:latest
|
|
|
|
WORKDIR /app
|
|
|
|
COPY . .
|
|
|
|
RUN apk add --no-cache bash
|
|
|
|
EXPOSE 8080
|
|
|
|
CMD ["sh", "-c", "while true; do sleep 30; done"]
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
# Main function to process all applications
|
|
main() {
|
|
echo "$(date): Starting Cloudron packaging process" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
# Read URLs from GitUrlList.txt
|
|
local urls=()
|
|
while IFS= read -r line; do
|
|
if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then
|
|
urls+=("$line")
|
|
fi
|
|
done < "$GIT_URL_LIST"
|
|
|
|
# Process applications in batches of 3 for parallel execution
|
|
local i=0
|
|
local total=${#urls[@]}
|
|
|
|
while [ $i -lt $total ]; do
|
|
# Process up to 3 applications in parallel
|
|
local end=$((i + 3))
|
|
[ $end -gt $total ] && end=$total
|
|
|
|
echo "$(date): Starting batch with applications $(printf '%s; ' "${urls[@]:i:3}")" >> "$WORKSPACES_DIR/packaging.log"
|
|
|
|
for ((j = i; j < end; j++)); do
|
|
echo "$(date): Starting packaging for ${urls[$j]}" >> "$WORKSPACES_DIR/packaging.log"
|
|
run_packaging_script "${urls[$j]}" &
|
|
done
|
|
|
|
# Wait for all background processes to complete
|
|
wait
|
|
|
|
# Update i for the next batch
|
|
i=$end
|
|
|
|
# Update progress summary in STATUS.md
|
|
local completed=$(grep -o "✅ COMPLETE" "$STATUS_FILE" | wc -l)
|
|
local failed=$(grep -o "🛑 FAILED" "$STATUS_FILE" | wc -l)
|
|
local in_progress=$(grep -o "🔄 IN PROGRESS" "$STATUS_FILE" | wc -l)
|
|
local pending=$((total - completed - failed - in_progress))
|
|
|
|
# Update summary section in STATUS.md
|
|
sed -i '/## Progress Summary/Q' "$STATUS_FILE"
|
|
cat >> "$STATUS_FILE" << EOF
|
|
## Progress Summary
|
|
- Total Applications: $total
|
|
- Completed: $completed ($(awk "BEGIN {printf \"%.0f\", $completed * 100 / $total}")%)
|
|
- In Progress: $in_progress ($(awk "BEGIN {printf \"%.0f\", $in_progress * 100 / $total}")%)
|
|
- Failed: $failed ($(awk "BEGIN {printf \"%.0f\", $failed * 100 / $total}")%)
|
|
- Pending: $pending ($(awk "BEGIN {printf \"%.0f\", $pending * 100 / $total}")%)
|
|
|
|
## Human Help Required
|
|
$(ls -1 "$HUMAN_HELP_DIR" 2>/dev/null || echo "None at the moment.")
|
|
|
|
## Last Updated
|
|
$(date)
|
|
EOF
|
|
done
|
|
|
|
echo "$(date): Completed Cloudron packaging process" >> "$WORKSPACES_DIR/packaging.log"
|
|
}
|
|
|
|
# Run the main function if script is executed directly
|
|
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
main "$@"
|
|
fi |