KNEL/TSYSDevStack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cloudron): update CloudronStack core components

Browse Source 
- Update CloudronStack/QWEN.md with latest development log information
- Update CloudronStack/collab/STATUS.md with current project status
- Update CloudronStack/output/master-control-script.sh with enhanced automation
- Update CloudronStack/output/package-functions.sh with improved packaging logic

These changes enhance the CloudronStack automation and packaging capabilities.
This commit is contained in:
Charles N Wyble - admin
2025-10-30 09:00:38 -05:00
parent d57db57018
commit 18d5a57868
4 changed files with 464 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
CloudronStack/output/master-control-script.sh
View File

@@ -10,11 +10,11 @@ set -o pipefail # Exit on pipe failures
# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUTPUT_DIR="$SCRIPT_DIR/output"
OUTPUT_DIR="$SCRIPT_DIR"
ARTIFACTS_DIR="$OUTPUT_DIR/CloudronPackages-Artifacts"
WORKSPACES_DIR="$OUTPUT_DIR/CloudronPackages-Workspaces"
STATUS_FILE="$SCRIPT_DIR/collab/STATUS.md"
GIT_URL_LIST="$SCRIPT_DIR/collab/GitUrlList.txt"
STATUS_FILE="$(dirname "$SCRIPT_DIR")/collab/STATUS.md"
GIT_URL_LIST="$(dirname "$SCRIPT_DIR")/collab/GitUrlList.txt"
HUMAN_HELP_DIR="$WORKSPACES_DIR/human-help-required"
MAX_RETRIES=5
LOG_FILE="$WORKSPACES_DIR/packaging.log"
@@ -42,8 +42,8 @@ perform_audit() {
# Count total, completed, failed, and in-progress applications
local total_count=$(grep -c "https://" "$GIT_URL_LIST" || echo 0)
local completed_count=$(grep -c "✅ COMPLETE" "$STATUS_FILE" || echo 0)
local failed_count=$(grep -c "🛑 FAILED" "$STATUS_FILE" || echo 0)
local completed_count=$(grep -c "✅ COMPLETE" "$STATUS_FILE" | grep -v "Progress Summary" || echo 0)
local failed_count=$(grep -c "🛑 FAILED" "$STATUS_FILE" | grep -v "Human Help Required" || echo 0)
local in_progress_count=$(grep -c "🔄 IN PROGRESS" "$STATUS_FILE" || echo 0)
local pending_count=$((total_count - completed_count - failed_count - in_progress_count))
@@ -68,19 +68,89 @@ perform_audit() {
log_message "INFO" "Audit process completed"
}
# Function to add a new Git URL to the list
add_git_url() {
local new_url=$1
local git_list_file=${2:-"$GIT_URL_LIST"}
if [[ -z "$new_url" ]]; then
log_message "ERROR" "No URL provided to add_git_url function"
return 1
fi
# Validate URL format
if [[ ! "$new_url" =~ ^https?:// ]]; then
log_message "ERROR" "Invalid URL format: $new_url"
return 1
fi
# Check if URL already exists in the file
if grep -Fxq "$new_url" "$git_list_file"; then
log_message "INFO" "URL already exists in $git_list_file: $new_url"
return 0
fi
# Add the URL to the file
echo "$new_url" >> "$git_list_file"
log_message "INFO" "Added new URL to $git_list_file: $new_url"
# Also update STATUS.md to include the new application
local repo_name=$(get_repo_name "$new_url")
local username_repo=$(get_username_repo "$new_url")
# Check if the application is already in STATUS.md
if ! grep -q "| $repo_name |" "$STATUS_FILE"; then
# Append the new application to the table in STATUS.md
sed -i "/## Applications Status/,/|-----|-----|-----|-----|/ {/|-----|-----|-----|-----|/a\| $repo_name | $new_url | ⏳ PENDING | |" "$STATUS_FILE"
log_message "INFO" "Added $repo_name to STATUS.md"
else
log_message "INFO" "Application $repo_name already exists in STATUS.md"
fi
return 0
}
# Function to add multiple Git URLs from a file
add_git_urls_from_file() {
local input_file=$1
local git_list_file=${2:-"$GIT_URL_LIST"}
if [[ ! -f "$input_file" ]]; then
log_message "ERROR" "Input file does not exist: $input_file"
return 1
fi
while IFS= read -r url; do
# Skip empty lines and comments
if [[ -n "$url" && ! "$url" =~ ^[[:space:]]*# ]]; then
add_git_url "$url" "$git_list_file"
fi
done < "$input_file"
log_message "INFO" "Finished processing URLs from $input_file"
}
# Function to update status in STATUS.md
update_status() {
local app_name=$1
local new_status=$2
local notes=${3:-""}
# Escape special characters for sed
local escaped_app_name=$(printf '%s\n' "$app_name" | sed 's/[[\.*^$()+?{|]/\\&/g')
local escaped_status=$(printf '%s\n' "$new_status" | sed 's/[[\.*^$()+?{|]/\\&/g')
local escaped_notes=$(printf '%s\n' "$notes" | sed 's/[[\.*^$()+?{|]/\\&/g' | sed 's/&/&amp;/g; s/</&lt;/g; s/>/&gt;/g')
# Validate inputs to prevent injection
if [[ -z "$app_name" ]] || [[ -z "$new_status" ]]; then
log_message "ERROR" "Empty app_name or new_status in update_status function"
return 1
fi
# Sanitize inputs to prevent injection
# Remove any pipe characters which would interfere with table format
local clean_app_name=$(printf '%s\n' "$app_name" | sed 's/|//g; s/[[\.*^$()+?{|]/\\&/g')
local clean_status=$(printf '%s\n' "$new_status" | sed 's/|//g; s/[[\.*^$()+?{|]/\\&/g')
local clean_notes=$(printf '%s\n' "$notes" | sed 's/|//g; s/[[\.*^$()+?{|]/\\&/g' | sed 's/&/&amp;/g; s/</&lt;/g; s/>/&gt;/g')
# Update status in the file - find the line with the app name and update its status
sed -i "s/^| $escaped_app_name |.*|.*|.*$/| $app_name |.*| $new_status | $escaped_notes |/" "$STATUS_FILE"
# Use a more targeted sed pattern to reduce chance of unintended matches
sed -i "s/^| $clean_app_name | \([^|]*\) | \([^|]*\) | \([^|]*\) |$/| $clean_app_name | \1 | $clean_status | $clean_notes |/" "$STATUS_FILE"
log_message "INFO" "Updated status for $app_name to $new_status"
}
@@ -90,38 +160,82 @@ get_repo_name() {
local url=$1
if [[ -z "$url" ]]; then
log_message "ERROR" "URL is empty in get_repo_name function"
echo "unknown-repo"
return 1
fi
local repo_part
repo_part=$(basename "$url")
repo_part=${repo_part%.git}
# Extract the basename more securely by using parameter expansion
# First remove any trailing slashes
local clean_url="${url%/}"
local repo_part="${clean_url##*/}"
repo_part="${repo_part%.git}"
# Sanitize the repo name to contain only valid characters
echo "$repo_part" | sed 's/[^a-zA-Z0-9._-]/-/g'
local sanitized=$(printf '%s\n' "$repo_part" | sed 's/[^a-zA-Z0-9._-]/-/g')
# Double-check to prevent path traversal
sanitized=$(printf '%s\n' "$sanitized" | sed 's/\.\.//g; s/\/\///g')
# Ensure the result is not empty
if [[ -z "$sanitized" ]] || [[ "$sanitized" == "." ]] || [[ "$sanitized" == ".." ]]; then
sanitized="unknown-repo-$(date +%s)"
fi
echo "$sanitized"
}
# Function to extract username/repo from URL for GitHub
# Function to extract username/repo from URL for GitHub/GitLab/other
get_username_repo() {
local url=$1
if [[ -z "$url" ]]; then
log_message "ERROR" "URL is empty in get_username_repo function"
echo "unknown-user/unknown-repo"
return 1
fi
if [[ "$url" == *"github.com"* ]]; then
# Clean the URL to prevent path traversal
local clean_url="${url#*://}" # Remove protocol
clean_url="${clean_url#*[email]*/}" # Remove potential user@host
if [[ "$clean_url" == *"github.com"* ]]; then
# Extract username/repo from GitHub URL
local path=${url#*github.com/}
local path=${clean_url#*github.com/}
path=${path%.git}
# Ensure we have a valid path
if [[ "$path" != *"/"* ]] || [[ "$path" == "/" ]]; then
# If there's no slash, it might be malformed, use repo name
path="unknown-user/$(get_repo_name "$url")"
else
# Sanitize the path to prevent directory traversal
path=$(printf '%s\n' "$path" | sed 's/\.\.//g; s/\/\///g')
fi
echo "$path"
elif [[ "$url" == *"gitlab.com"* ]]; then
elif [[ "$clean_url" == *"gitlab.com"* ]]; then
# Extract username/repo from GitLab URL
local path=${url#*gitlab.com/}
local path=${clean_url#*gitlab.com/}
path=${path%.git}
# Ensure we have a valid path
if [[ "$path" != *"/"* ]] || [[ "$path" == "/" ]]; then
# If there's no slash, it might be malformed, use repo name
path="unknown-user/$(get_repo_name "$url")"
else
# Sanitize the path to prevent directory traversal
path=$(printf '%s\n' "$path" | sed 's/\.\.//g; s/\/\///g')
fi
echo "$path"
else
# For other URLs, just return the repo name
get_repo_name "$url"
# For other URLs, try to extract pattern user/repo
local path=${clean_url#*/} # Remove host part
if [[ "$path" == *"/"* ]]; then
path=${path%.git}
# Sanitize the path to prevent directory traversal
path=$(printf '%s\n' "$path" | sed 's/\.\.//g; s/\/\///g')
else
# If no slash, use a generic format
local repo=$(get_repo_name "$url")
path="unknown-user/$repo"
fi
echo "$path"
fi
}
@@ -173,7 +287,7 @@ run_packaging_script() {
echo "$(date): Attempt $attempt/$MAX_RETRIES for $repo_name" >> "$WORKSPACES_DIR/packaging.log"
# Capture the output and error of the packaging function
if package_application "$repo_name" "$username_repo" "$workspace_dir" "$artifact_dir" 2>"$workspace_dir/error.log"; then
if package_application "$repo_name" "$username_repo" "$workspace_dir" "$artifact_dir" "$url" 2>"$workspace_dir/error.log"; then
success=1
update_status "$repo_name" "✅ COMPLETE" "Packaged successfully on attempt $attempt"
echo "$(date): Successfully packaged $repo_name on attempt $attempt" >> "$WORKSPACES_DIR/packaging.log"
@@ -207,11 +321,12 @@ package_application() {
local username_repo=$2
local workspace_dir=$3
local artifact_dir=$4
local url=${5:-"https://github.com/unknown-user/$repo_name"} # Default URL if not provided
local repo_path="$workspace_dir/repo"
# Use the function library to detect and package the application
detect_and_package "$repo_name" "$repo_path" "$artifact_dir"
detect_and_package "$repo_name" "$repo_path" "$artifact_dir" "$url"
}
# Function to create a Dockerfile based on the application type
@@ -301,19 +416,46 @@ EOF
fi
}
# Function to load URLs from Git URL list file
load_git_urls() {
local git_list_file=${1:-"$GIT_URL_LIST"}
local urls=()
if [[ ! -f "$git_list_file" ]]; then
log_message "ERROR" "Git URL list file does not exist: $git_list_file"
return 1
fi
while IFS= read -r line; do
# Skip empty lines and comments
if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then
# Validate that the line looks like a URL
if [[ "$line" =~ ^https?:// ]]; then
urls+=("$line")
else
log_message "WARN" "Invalid URL format skipped: $line"
fi
fi
done < "$git_list_file"
# Print the urls array to stdout so the caller can capture it
printf '%s\n' "${urls[@]}"
}
# Main function to process all applications
main() {
log_message "INFO" "Starting Cloudron packaging process"
# Read URLs from GitUrlList.txt
local urls=()
while IFS= read -r line; do
if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then
urls+=("$line")
fi
done < "$GIT_URL_LIST"
# Validate that required files exist
if [[ ! -f "$SCRIPT_DIR/package-functions.sh" ]]; then
log_message "ERROR" "Package functions file does not exist: $SCRIPT_DIR/package-functions.sh"
exit 1
fi
local total=${#urls[@]}
# Load URLs from the git list file
local url_list
mapfile -t url_list < <(load_git_urls)
local total=${#url_list[@]}
log_message "INFO" "Found $total URLs to process"
# Process applications in batches of 3 for parallel execution
@@ -324,11 +466,11 @@ main() {
local end=$((i + 3))
[ $end -gt $total ] && end=$total
log_message "INFO" "Starting batch with applications $(printf '%s; ' "${urls[@]:i:3}")"
log_message "INFO" "Starting batch with applications $(printf '%s; ' "${url_list[@]:i:3}")"
for ((j = i; j < end; j++)); do
log_message "INFO" "Starting packaging for ${urls[$j]}"
run_packaging_script "${urls[$j]}" &
log_message "INFO" "Starting packaging for ${url_list[$j]}"
run_packaging_script "${url_list[$j]}" &
done
# Wait for all background processes to complete
@@ -346,6 +488,9 @@ main() {
local in_progress=$(grep -o "🔄 IN PROGRESS" "$STATUS_FILE" | wc -l)
local pending=$((total - completed - failed - in_progress))
# Ensure we don't have negative pending due to counting issues
[ $pending -lt 0 ] && pending=0
# Update summary section in STATUS.md
sed -i '/## Progress Summary/Q' "$STATUS_FILE"
cat >> "$STATUS_FILE" << EOF