Implement a restart facility.

If you select to debug ct-ng, then you have two new options:
 - DEBUG_CT_PAUSE_STEPS : pause between every steps,
 - DEBUG_CT_SAVE_STEPS  : save state between every steps.
To restart a saved state, just set the RESTART make variable when calling make:
  - make RESTART=<step_name>

Makefile

@@ -9,6 +9,8 @@ export CT_TOP_DIR=$(shell pwd)
 # This is crosstool-ng version string
 export CT_VERSION=$(shell cat $(CT_TOP_DIR)/version)
 
+export CT_RESTART=$(RESTART)
+
 .PHONY: all
 all: build
 
@@ -32,6 +34,7 @@ include $(CT_TOP_DIR)/tools/Makefile
 
 help::
 	@echo  'Execute "make" or "make all" to build all targets marked with [*]'
+	@echo  'Pass RESTART=<step> to restart a previously saved step (read doc first!)'
 
 .config: $(CONFIG_FILES) $(CT_TOP_DIR)/config/debug.in
 	@make oldconfig

config/global.in

@@ -1,7 +1,16 @@
 # Overall toolchain configuration: paths, jobs, etc...
 
+# Ah, this option is here to break the dependency tracking, and allow
+# dependent option to line-up with the options they depend on ,rather
+# than being indented
+config FOOBAR
+    bool
+    default n
+
 menu "Paths and misc options"
 
+comment "Crosstool behavior"
+
 config EXPERIMENTAL
     bool
     prompt "Try features marked as EXPERIMENTAL"
@@ -26,6 +35,64 @@ config OBSOLETE
       Such obsolete features are the use of old kernel headers, old
       gcc versions, etc...
 
+config DEBUG_CT
+    bool
+    prompt "Debug crosstool-NG"
+    default n
+    help
+      Say 'y' here to get some debugging options
+
+if DEBUG_CT
+
+config DEBUG_CT_PAUSE_STEPS
+    bool
+    prompt "Pause between every steps"
+    default n
+    help
+      Say 'y' if you intend to attend the build, and want to investigate
+      the result of each steps before running the next one.
+
+config DEBUG_CT_SAVE_STEPS
+    bool
+    prompt "Save intermediate steps"
+    default n
+    help
+      If you say 'y' here, then you will be able to restart crosstool-NG at
+      any step by typing:
+          make RESTART=<step_nane>
+      
+      where step_name is one of:
+        - kernel_hdr
+        - binutils
+        - libc_hdr
+        - cc_core
+        - libfloat
+        - libc
+        - cc
+        - libc_fin
+        - debug
+      
+      It is not currently possible to rstart at any of the debug facility.
+      They are treated a s a whole.
+
+config DEBUG_CT_SAVE_STEPS_GZIP
+    bool
+    prompt "gzip saved states"
+    default y
+    depends on DEBUG_CT_SAVE_STEPS
+    help
+      If you are tight on space, then you can ask to gzip the saved states
+      tarballs. On the other hand, this takes some longer time...
+      
+      To lose as less time as possible, the gzip process is done with a low
+      compression ratio (-3), which gives roughly 70% gain in size. Going
+      further doesn't gain much, and takes far more time (believe me, I've
+      got figures here! :-) ).
+
+endif
+
+comment "Build behavior"
+
 config PARALLEL_JOBS
     int
     prompt "Number of parallel jobs"
@@ -157,6 +224,8 @@ config ONLY_DOWNLOAD
       
       Usefull to pre-retrieve the tarballs before going off-line.
 
+config FOOBAR
+
 comment "Extracting"
     depends on ! ONLY_DOWNLOAD
 

scripts/crosstool.sh

@@ -28,10 +28,10 @@ CT_STAR_DATE=`CT_DoDate +%s%N`
 CT_STAR_DATE_HUMAN=`CT_DoDate +%Y%m%d.%H%M%S`
 
 # Log policy:
-#  - what goes to the log file goes to fd #1 (stdout)
+#  - first of all, save stdout so we can see the live logs: fd #6
-#  - what goes to the screen goes to fd #6
-tmp_log_file="${CT_TOP_DIR}/$$.log"
 exec 6>&1
+#  - then point stdout to the log file (temporary for now)
+tmp_log_file="${CT_TOP_DIR}/$$.log"
 exec >>"${tmp_log_file}"
 
 # Are we configured? We'll need that later...
@@ -54,6 +54,13 @@ CT_EndStep
 # Some sanity checks in the environment and needed tools
 CT_DoLog INFO "Checking environment sanity"
 
+# First of really first of really all, **must not** move lower than here!
+if [ -n "${CT_RESTART}" -a -z "${CT_DEBUG_CT_SAVE_STEPS}" ]; then
+    CT_DoLog ERROR "You asked to restart a non-restartable build"
+    CT_DoLog ERROR "This happened because you didn't set CT_DEBUG_CT_SAVE_STEPS in the config options"
+    CT_Abort "I will stop here to avoid any carnage"
+fi
+
 # Enable known ordering of files in directory listings:
 CT_Test "Crosstool-NG might not work as expected with LANG=\"${LANG}\"" -n "${LANG}"
 case "${LC_COLLATE},${LC_ALL}" in
@@ -108,6 +115,11 @@ CT_TARBALLS_DIR="${CT_TOP_DIR}/targets/tarballs"
 CT_SRC_DIR="${CT_TOP_DIR}/targets/${CT_TARGET}/src"
 CT_BUILD_DIR="${CT_TOP_DIR}/targets/${CT_TARGET}/build"
 CT_DEBUG_INSTALL_DIR="${CT_INSTALL_DIR}/${CT_TARGET}/debug-root"
+# Note: we'll always install the core compiler in its own directory, so as to
+# not mix the two builds: core and final. Anyway, its generic, wether we use
+# a different compiler as core, or not.
+CT_CC_CORE_PREFIX_DIR="${CT_BUILD_DIR}/${CT_CC}-core"
+CT_STATE_DIR="${CT_TOP_DIR}/targets/${CT_TARGET}/state"
 
 # Make all path absolute, it so much easier!
 CT_LOCAL_TARBALLS_DIR="`CT_MakeAbsolutePath \"${CT_LOCAL_TARBALLS_DIR}\"`"
@@ -140,6 +152,15 @@ CT_TOOLCHAIN_ID="crosstool-${CT_VERSION} build ${CT_STAR_DATE_HUMAN} by ${CT_SYS
 
 CT_DoLog EXTRA "Preparing working directories"
 
+# Ah! The build directory shall be eradicated, even if we restart!
+if [ -d "${CT_BUILD_DIR}" ]; then
+    mv "${CT_BUILD_DIR}" "${CT_BUILD_DIR}.$$"
+    chmod -R u+w "${CT_BUILD_DIR}.$$"
+    nohup rm -rf "${CT_BUILD_DIR}.$$" >/dev/null 2>&1 &
+fi
+
+# Don't eradicate directories if we need to restart
+if [ -z "${CT_RESTART}" ]; then
     # Get rid of pre-existing installed toolchain and previous build directories.
     # We need to do that _before_ we can safely log, because the log file will
     # most probably be in the toolchain directory.
@@ -153,11 +174,6 @@ if [ "${CT_FORCE_EXTRACT}" = "y" -a -d "${CT_SRC_DIR}" ]; then
         chmod -R u+w "${CT_SRC_DIR}.$$"
         nohup rm -rf "${CT_SRC_DIR}.$$" >/dev/null 2>&1 &
     fi
-if [ -d "${CT_BUILD_DIR}" ]; then
-    mv "${CT_BUILD_DIR}" "${CT_BUILD_DIR}.$$"
-    chmod -R u+w "${CT_BUILD_DIR}.$$"
-    nohup rm -rf "${CT_BUILD_DIR}.$$" >/dev/null 2>&1 &
-fi
     if [ -d "${CT_INSTALL_DIR}" ]; then
         mv "${CT_INSTALL_DIR}" "${CT_INSTALL_DIR}.$$"
         chmod -R u+w "${CT_INSTALL_DIR}.$$"
@@ -168,19 +184,30 @@ if [ -d "${CT_DEBUG_INSTALL_DIR}" ]; then
         chmod -R u+w "${CT_DEBUG_INSTALL_DIR}.$$"
         nohup rm -rf "${CT_DEBUG_INSTALL_DIR}.$$" >/dev/null 2>&1 &
     fi
+    # In case we start anew, get rid of the previously saved state directory
+    if [ -d "${CT_STATE_DIR}" ]; then
+        mv "${CT_STATE_DIR}" "${CT_STATE_DIR}.$$"
+        chmod -R u+w "${CT_STATE_DIR}.$$"
+        nohup rm -rf "${CT_STATE_DIR}.$$" >/dev/null 2>&1 &
+    fi
+fi
 
-# Note: we'll always install the core compiler in its own directory, so as to
+# Create the directories we'll use, even if restarting: it does no harm to
-# not mix the two builds: core and final. Anyway, its generic, wether we use
+# create already existent directories, and CT_BUILD_DIR needs to be created
-# a different compiler as core, or not.
+# anyway
-CT_CC_CORE_PREFIX_DIR="${CT_BUILD_DIR}/${CT_CC}-core"
-
-# Create the directories we'll use
 mkdir -p "${CT_TARBALLS_DIR}"
 mkdir -p "${CT_SRC_DIR}"
 mkdir -p "${CT_BUILD_DIR}"
 mkdir -p "${CT_INSTALL_DIR}"
+mkdir -p "${CT_PREFIX_DIR}"
 mkdir -p "${CT_DEBUG_INSTALL_DIR}"
 mkdir -p "${CT_CC_CORE_PREFIX_DIR}"
+mkdir -p "${CT_STATE_DIR}"
+
+# Kludge: CT_INSTALL_DIR and CT_PREFIX_DIR might have grown read-only if
+# the previous build was successfull. To ba able to move the logfile there,
+# switch them back to read/write
+chmod -R u+w "${CT_INSTALL_DIR}" "${CT_PREFIX_DIR}"
 
 # Redirect log to the actual log file now we can
 # It's quite understandable that the log file will be installed in the install
@@ -191,15 +218,19 @@ case "${CT_LOG_TO_FILE},${CT_LOG_FILE}" in
     ,*)   rm -f "${tmp_log_file}"
           ;;
     y,/*) mkdir -p "`dirname \"${CT_LOG_FILE}\"`"
-          mv "${tmp_log_file}" "${CT_LOG_FILE}"
+          cat "${tmp_log_file}" >>"${CT_LOG_FILE}"
+          rm -f "${tmp_log_file}"
           exec >>"${CT_LOG_FILE}"
           ;;
     y,*)  mkdir -p "`pwd`/`dirname \"${CT_LOG_FILE}\"`"
-          mv "${tmp_log_file}" "`pwd`/${CT_LOG_FILE}"
+          cat "${tmp_log_file}" >>"`pwd`/${CT_LOG_FILE}"
+          rm -f "${tmp_log_file}"
           exec >>"${CT_LOG_FILE}"
           ;;
 esac
 
+# Setting up the rest of the environment only is not restarting
+if [ -z "${CT_RESTART}" ]; then
     # Determine build system if not set by the user
     CT_Test "You did not specify the build system. That's OK, I can guess..." -z "${CT_BUILD}"
     CT_BUILD="`${CT_TOP_DIR}/tools/config.sub \"${CT_BUILD:-\`${CT_TOP_DIR}/tools/config.guess\`}\"`"
@@ -271,11 +302,13 @@ esac
     # Do that:
     CT_DoLog EXTRA "Making build system tools available"
     mkdir -p "${CT_PREFIX_DIR}/bin"
-for tool in ar gcc; do
+    for tool in ar as dlltool gcc g++ gnatbind gnatmake ld nm ranlib strip windres objcopy objdump; do
+        if [ -n "`which ${tool}`" ]; then
             ln -s "`which ${tool}`" "${CT_PREFIX_DIR}/bin/${CT_BUILD}-${tool}"
             case "${CT_TOOLCHAIN_TYPE}" in
                 cross|native)   ln -s "`which ${tool}`" "${CT_PREFIX_DIR}/bin/${CT_HOST}-${tool}";;
             esac
+        fi
     done
 
     # Ha. cygwin host have an .exe suffix (extension) for executables.
@@ -311,6 +344,7 @@ CT_DoLog EXTRA "  host   = ${CT_HOST}"
     CT_DoLog EXTRA "  target = ${CT_TARGET}"
     set |egrep '^CT_.+=' |sort |CT_DoLog DEBUG
     CT_EndStep
+fi
 
 # Include sub-scripts instead of calling them: that way, we do not have to
 # export any variable, nor re-parse the configuration and functions files.
@@ -322,7 +356,7 @@ CT_EndStep
 . "${CT_TOP_DIR}/scripts/build/cc_${CT_CC}.sh"
 . "${CT_TOP_DIR}/scripts/build/debug.sh"
 
-# Now for the job by itself. Go have a coffee!
+if [ -z "${CT_RESTART}" ]; then
     CT_DoStep INFO "Retrieving needed toolchain components' tarballs"
     do_kernel_get
     do_binutils_get
@@ -347,19 +381,40 @@ if [ "${CT_ONLY_DOWNLOAD}" != "y" ]; then
         do_cc_extract
         do_debug_extract
         CT_EndStep
+    fi
+fi
 
-    if [ "${CT_ONLY_EXTRACT}" != "y" ]; then
+# Now for the job by itself. Go have a coffee!
-        do_libc_check_config
+if [ "${CT_ONLY_DOWNLOAD}" != "y" -a "${CT_ONLY_EXTRACT}" != "y" ]; then
-        do_kernel_check_config
+    # Because of CT_RESTART, this becomes quite complex
-        do_kernel_headers
+    [ -n "${CT_RESTART}" ] && do_it=0 || do_it=1
-        do_binutils
+    for step in libc_check_config       \
-        do_libc_headers
+                kernel_check_config     \
-        do_cc_core
+                kernel_headers          \
-        do_libfloat
+                binutils                \
-        do_libc
+                libc_headers            \
-        do_cc
+                cc_core                 \
-        do_libc_finish
+                libfloat                \
-        do_debug
+                libc                    \
+                cc                      \
+                libc_finish             \
+                debug                   \
+                ; do
+        if [ ${do_it} -eq 0 ]; then
+            if [ "${CT_RESTART}" = "${step}" ]; then
+                CT_DoLoadState "${step}"
+                do_it=1
+            fi
+        else
+            CT_DoSaveState ${step}
+        fi
+        if [ ${do_it} -eq 1 ]; then
+            do_${step}
+            if [ "${CTDEBUG_CT_PAUSE_STEPS}" = "y" ]; then
+                CT_DoPause "Step \"${step}\" finished"
+            fi
+        fi
+    done
 
     # Create the aliases to the target tools
     if [ -n "${CT_TARGET_ALIAS}" ]; then
@@ -381,7 +436,6 @@ if [ "${CT_ONLY_DOWNLOAD}" != "y" ]; then
         rm -rf "${CT_DEBUG_INSTALL_DIR}/"{,usr/}{man,info}
     fi
 fi
-fi
 
 # OK, now we're done, set the toolchain read-only
 # Don't log, the log file may become read-only any moment...

scripts/functions

@@ -304,7 +304,7 @@ CT_GetFile() {
         if [ -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" -a \
              "${CT_FORCE_DOWNLOAD}" != "y" ]; then
             CT_DoLog EXTRA "Copying \"${file}\" from local copy"
-            cp -v "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog DEBUG
+            cp -v "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL
             return 0
         fi
     done
@@ -460,3 +460,112 @@ CT_DoBuildTargetTriplet() {
     esac
     CT_TARGET="`${CT_TOP_DIR}/tools/config.sub ${CT_TARGET}`"
 }
+
+# This function does pause the build until the user strikes "Return"
+# Usage: CT_DoPause [optional_message]
+CT_DoPause() {
+    local foo
+    local message="${1:-Pausing for your pleasure}"
+    CT_DoLog INFO "${message}"
+    read -p "Press \"Enter\" to continue, or Ctrl-C to stop..." foo >&6
+    return 0
+}
+
+# This function saves the state of the toolchain to be able to restart
+# at any one point
+# Usage: CT_DoSaveState <next_step_name>
+CT_DoSaveState() {
+	[ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0
+    local state_name="$1"
+    local state_dir="${CT_STATE_DIR}/${state_name}"
+
+    CT_DoLog DEBUG "Saving state to restart at step \"${state_name}\"..."
+    rm -rf "${state_dir}"
+    mkdir -p "${state_dir}"
+
+    case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
+        y)  tar_opt=czf; tar_ext=".tar.gz";;
+        *)  tar_opt=cf;  tar_ext=".tar";;
+    esac
+
+    CT_DoLog DEBUG "  Saving environment and aliases"
+    # We must omit shell functions
+    # 'isgrep' is here because I don't seem to
+    # be able to remove the functions names.
+    set |awk '
+         BEGIN { _p = 1; }
+         $0~/^[^ ] ()/ { _p = 0; }
+         _p == 1
+         $0 == "}" { _p = 1; }
+         ' |egrep -v '^[^ ]+ \(\)' >"${state_dir}/env.sh"
+
+    CT_DoLog DEBUG "  Saving CT_CC_CORE_PREFIX_DIR=\"${CT_CC_CORE_PREFIX_DIR}\""
+    CT_Pushd "${CT_CC_CORE_PREFIX_DIR}"
+    tar ${tar_opt} "${state_dir}/cc_core_prefix_dir${tar_ext}" .
+    CT_Popd
+
+    CT_DoLog DEBUG "  Saving CT_PREFIX_DIR=\"${CT_PREFIX_DIR}\""
+    CT_Pushd "${CT_PREFIX_DIR}"
+    tar ${tar_opt} "${state_dir}/prefix_dir${tar_ext}" .
+    CT_Popd
+
+    if [ "${CT_LOG_TO_FILE}" = "y" ]; then
+        CT_DoLog DEBUG "  Saving log file"
+        exec >/dev/null
+        case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
+            y)  gzip -3 -c "${CT_LOG_FILE}"  >"${state_dir}/log.gz";;
+            *)  cat "${CT_LOG_FILE}" >"${state_dir}/log";;
+        esac
+        exec >>"${CT_LOG_FILE}"
+    fi
+}
+
+# This functions restores a previously saved state
+# Usage: CT_DoLoadState <state_name>
+CT_DoLoadState(){
+    local state_name="$1"
+    local state_dir="${CT_STATE_DIR}/${state_name}"
+
+    # We need to do something special with the log file!
+    if [ "${CT_LOG_TO_FILE}" = "y" ]; then
+        exec >"${state_dir}/tail.log"
+    fi
+    CT_DoLog DEBUG "Restoring state at step \"${state_name}\"..."
+
+    case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
+        y)  tar_opt=xzf; tar_ext=".tar.gz";;
+        *)  tar_opt=cf;  tar_ext=".tar";;
+    esac
+
+    CT_DoLog DEBUG "  Removing previous build directories"
+    chmod -R u+rwX "${CT_PREFIX_DIR}" "${CT_CC_CORE_PREFIX_DIR}"
+    rm -rf         "${CT_PREFIX_DIR}" "${CT_CC_CORE_PREFIX_DIR}"
+    mkdir -p       "${CT_PREFIX_DIR}" "${CT_CC_CORE_PREFIX_DIR}"
+
+    CT_DoLog DEBUG "  Restoring CT_PREFIX_DIR=\"${CT_PREFIX_DIR}\""
+    CT_Pushd "${CT_PREFIX_DIR}"
+    tar ${tar_opt} "${state_dir}/prefix_dir${tar_ext}"
+    CT_Popd
+
+    CT_DoLog DEBUG "  Restoring CT_CC_CORE_PREFIX_DIR=\"${CT_CC_CORE_PREFIX_DIR}\""
+    CT_Pushd "${CT_CC_CORE_PREFIX_DIR}"
+    tar ${tar_opt} "${state_dir}/cc_core_prefix_dir${tar_ext}"
+    CT_Popd
+
+    # Restore the environment, discarding any error message
+    # (for example, read-only bash internals)
+    CT_DoLog DEBUG "  Restoring environment"
+    . "${state_dir}/env.sh" >/dev/null 2>&1 || true
+
+    if [ "${CT_LOG_TO_FILE}" = "y" ]; then
+        CT_DoLog DEBUG "  Restoring log file"
+        exec >/dev/null
+        case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
+            y)  zcat "${state_dir}/log.gz" >"${CT_LOG_FILE}";;
+            *)  cat "${state_dir}/log" >"${CT_LOG_FILE}";;
+        esac
+        cat "${state_dir}/tail.log" >>"${CT_LOG_FILE}"
+        exec >>"${CT_LOG_FILE}"
+        rm -f "${state_dir}/tail.log"
+    fi
+}