Implement a restart facility.

If you select to debug ct-ng, then you have two new options:
 - DEBUG_CT_PAUSE_STEPS : pause between every steps,
 - DEBUG_CT_SAVE_STEPS  : save state between every steps.
To restart a saved state, just set the RESTART make variable when calling make:
  - make RESTART=<step_name>
This commit is contained in:
Yann E. MORIN"
2007-05-22 20:46:07 +00:00
parent 0c28252021
commit cb0d1fef8c
4 changed files with 422 additions and 187 deletions

View File

@ -9,6 +9,8 @@ export CT_TOP_DIR=$(shell pwd)
# This is crosstool-ng version string
export CT_VERSION=$(shell cat $(CT_TOP_DIR)/version)
export CT_RESTART=$(RESTART)
.PHONY: all
all: build
@ -32,6 +34,7 @@ include $(CT_TOP_DIR)/tools/Makefile
help::
@echo 'Execute "make" or "make all" to build all targets marked with [*]'
@echo 'Pass RESTART=<step> to restart a previously saved step (read doc first!)'
.config: $(CONFIG_FILES) $(CT_TOP_DIR)/config/debug.in
@make oldconfig

View File

@ -1,7 +1,16 @@
# Overall toolchain configuration: paths, jobs, etc...
# Ah, this option is here to break the dependency tracking, and allow
# dependent option to line-up with the options they depend on ,rather
# than being indented
config FOOBAR
bool
default n
menu "Paths and misc options"
comment "Crosstool behavior"
config EXPERIMENTAL
bool
prompt "Try features marked as EXPERIMENTAL"
@ -26,6 +35,64 @@ config OBSOLETE
Such obsolete features are the use of old kernel headers, old
gcc versions, etc...
config DEBUG_CT
bool
prompt "Debug crosstool-NG"
default n
help
Say 'y' here to get some debugging options
if DEBUG_CT
config DEBUG_CT_PAUSE_STEPS
bool
prompt "Pause between every steps"
default n
help
Say 'y' if you intend to attend the build, and want to investigate
the result of each steps before running the next one.
config DEBUG_CT_SAVE_STEPS
bool
prompt "Save intermediate steps"
default n
help
If you say 'y' here, then you will be able to restart crosstool-NG at
any step by typing:
make RESTART=<step_nane>
where step_name is one of:
- kernel_hdr
- binutils
- libc_hdr
- cc_core
- libfloat
- libc
- cc
- libc_fin
- debug
It is not currently possible to rstart at any of the debug facility.
They are treated a s a whole.
config DEBUG_CT_SAVE_STEPS_GZIP
bool
prompt "gzip saved states"
default y
depends on DEBUG_CT_SAVE_STEPS
help
If you are tight on space, then you can ask to gzip the saved states
tarballs. On the other hand, this takes some longer time...
To lose as less time as possible, the gzip process is done with a low
compression ratio (-3), which gives roughly 70% gain in size. Going
further doesn't gain much, and takes far more time (believe me, I've
got figures here! :-) ).
endif
comment "Build behavior"
config PARALLEL_JOBS
int
prompt "Number of parallel jobs"
@ -157,6 +224,8 @@ config ONLY_DOWNLOAD
Usefull to pre-retrieve the tarballs before going off-line.
config FOOBAR
comment "Extracting"
depends on ! ONLY_DOWNLOAD

View File

@ -28,10 +28,10 @@ CT_STAR_DATE=`CT_DoDate +%s%N`
CT_STAR_DATE_HUMAN=`CT_DoDate +%Y%m%d.%H%M%S`
# Log policy:
# - what goes to the log file goes to fd #1 (stdout)
# - what goes to the screen goes to fd #6
tmp_log_file="${CT_TOP_DIR}/$$.log"
# - first of all, save stdout so we can see the live logs: fd #6
exec 6>&1
# - then point stdout to the log file (temporary for now)
tmp_log_file="${CT_TOP_DIR}/$$.log"
exec >>"${tmp_log_file}"
# Are we configured? We'll need that later...
@ -54,6 +54,13 @@ CT_EndStep
# Some sanity checks in the environment and needed tools
CT_DoLog INFO "Checking environment sanity"
# First of really first of really all, **must not** move lower than here!
if [ -n "${CT_RESTART}" -a -z "${CT_DEBUG_CT_SAVE_STEPS}" ]; then
CT_DoLog ERROR "You asked to restart a non-restartable build"
CT_DoLog ERROR "This happened because you didn't set CT_DEBUG_CT_SAVE_STEPS in the config options"
CT_Abort "I will stop here to avoid any carnage"
fi
# Enable known ordering of files in directory listings:
CT_Test "Crosstool-NG might not work as expected with LANG=\"${LANG}\"" -n "${LANG}"
case "${LC_COLLATE},${LC_ALL}" in
@ -108,6 +115,11 @@ CT_TARBALLS_DIR="${CT_TOP_DIR}/targets/tarballs"
CT_SRC_DIR="${CT_TOP_DIR}/targets/${CT_TARGET}/src"
CT_BUILD_DIR="${CT_TOP_DIR}/targets/${CT_TARGET}/build"
CT_DEBUG_INSTALL_DIR="${CT_INSTALL_DIR}/${CT_TARGET}/debug-root"
# Note: we'll always install the core compiler in its own directory, so as to
# not mix the two builds: core and final. Anyway, its generic, wether we use
# a different compiler as core, or not.
CT_CC_CORE_PREFIX_DIR="${CT_BUILD_DIR}/${CT_CC}-core"
CT_STATE_DIR="${CT_TOP_DIR}/targets/${CT_TARGET}/state"
# Make all path absolute, it so much easier!
CT_LOCAL_TARBALLS_DIR="`CT_MakeAbsolutePath \"${CT_LOCAL_TARBALLS_DIR}\"`"
@ -140,6 +152,15 @@ CT_TOOLCHAIN_ID="crosstool-${CT_VERSION} build ${CT_STAR_DATE_HUMAN} by ${CT_SYS
CT_DoLog EXTRA "Preparing working directories"
# Ah! The build directory shall be eradicated, even if we restart!
if [ -d "${CT_BUILD_DIR}" ]; then
mv "${CT_BUILD_DIR}" "${CT_BUILD_DIR}.$$"
chmod -R u+w "${CT_BUILD_DIR}.$$"
nohup rm -rf "${CT_BUILD_DIR}.$$" >/dev/null 2>&1 &
fi
# Don't eradicate directories if we need to restart
if [ -z "${CT_RESTART}" ]; then
# Get rid of pre-existing installed toolchain and previous build directories.
# We need to do that _before_ we can safely log, because the log file will
# most probably be in the toolchain directory.
@ -153,11 +174,6 @@ if [ "${CT_FORCE_EXTRACT}" = "y" -a -d "${CT_SRC_DIR}" ]; then
chmod -R u+w "${CT_SRC_DIR}.$$"
nohup rm -rf "${CT_SRC_DIR}.$$" >/dev/null 2>&1 &
fi
if [ -d "${CT_BUILD_DIR}" ]; then
mv "${CT_BUILD_DIR}" "${CT_BUILD_DIR}.$$"
chmod -R u+w "${CT_BUILD_DIR}.$$"
nohup rm -rf "${CT_BUILD_DIR}.$$" >/dev/null 2>&1 &
fi
if [ -d "${CT_INSTALL_DIR}" ]; then
mv "${CT_INSTALL_DIR}" "${CT_INSTALL_DIR}.$$"
chmod -R u+w "${CT_INSTALL_DIR}.$$"
@ -168,19 +184,30 @@ if [ -d "${CT_DEBUG_INSTALL_DIR}" ]; then
chmod -R u+w "${CT_DEBUG_INSTALL_DIR}.$$"
nohup rm -rf "${CT_DEBUG_INSTALL_DIR}.$$" >/dev/null 2>&1 &
fi
# In case we start anew, get rid of the previously saved state directory
if [ -d "${CT_STATE_DIR}" ]; then
mv "${CT_STATE_DIR}" "${CT_STATE_DIR}.$$"
chmod -R u+w "${CT_STATE_DIR}.$$"
nohup rm -rf "${CT_STATE_DIR}.$$" >/dev/null 2>&1 &
fi
fi
# Note: we'll always install the core compiler in its own directory, so as to
# not mix the two builds: core and final. Anyway, its generic, wether we use
# a different compiler as core, or not.
CT_CC_CORE_PREFIX_DIR="${CT_BUILD_DIR}/${CT_CC}-core"
# Create the directories we'll use
# Create the directories we'll use, even if restarting: it does no harm to
# create already existent directories, and CT_BUILD_DIR needs to be created
# anyway
mkdir -p "${CT_TARBALLS_DIR}"
mkdir -p "${CT_SRC_DIR}"
mkdir -p "${CT_BUILD_DIR}"
mkdir -p "${CT_INSTALL_DIR}"
mkdir -p "${CT_PREFIX_DIR}"
mkdir -p "${CT_DEBUG_INSTALL_DIR}"
mkdir -p "${CT_CC_CORE_PREFIX_DIR}"
mkdir -p "${CT_STATE_DIR}"
# Kludge: CT_INSTALL_DIR and CT_PREFIX_DIR might have grown read-only if
# the previous build was successfull. To ba able to move the logfile there,
# switch them back to read/write
chmod -R u+w "${CT_INSTALL_DIR}" "${CT_PREFIX_DIR}"
# Redirect log to the actual log file now we can
# It's quite understandable that the log file will be installed in the install
@ -191,15 +218,19 @@ case "${CT_LOG_TO_FILE},${CT_LOG_FILE}" in
,*) rm -f "${tmp_log_file}"
;;
y,/*) mkdir -p "`dirname \"${CT_LOG_FILE}\"`"
mv "${tmp_log_file}" "${CT_LOG_FILE}"
cat "${tmp_log_file}" >>"${CT_LOG_FILE}"
rm -f "${tmp_log_file}"
exec >>"${CT_LOG_FILE}"
;;
y,*) mkdir -p "`pwd`/`dirname \"${CT_LOG_FILE}\"`"
mv "${tmp_log_file}" "`pwd`/${CT_LOG_FILE}"
cat "${tmp_log_file}" >>"`pwd`/${CT_LOG_FILE}"
rm -f "${tmp_log_file}"
exec >>"${CT_LOG_FILE}"
;;
esac
# Setting up the rest of the environment only is not restarting
if [ -z "${CT_RESTART}" ]; then
# Determine build system if not set by the user
CT_Test "You did not specify the build system. That's OK, I can guess..." -z "${CT_BUILD}"
CT_BUILD="`${CT_TOP_DIR}/tools/config.sub \"${CT_BUILD:-\`${CT_TOP_DIR}/tools/config.guess\`}\"`"
@ -271,11 +302,13 @@ esac
# Do that:
CT_DoLog EXTRA "Making build system tools available"
mkdir -p "${CT_PREFIX_DIR}/bin"
for tool in ar gcc; do
for tool in ar as dlltool gcc g++ gnatbind gnatmake ld nm ranlib strip windres objcopy objdump; do
if [ -n "`which ${tool}`" ]; then
ln -s "`which ${tool}`" "${CT_PREFIX_DIR}/bin/${CT_BUILD}-${tool}"
case "${CT_TOOLCHAIN_TYPE}" in
cross|native) ln -s "`which ${tool}`" "${CT_PREFIX_DIR}/bin/${CT_HOST}-${tool}";;
esac
fi
done
# Ha. cygwin host have an .exe suffix (extension) for executables.
@ -311,6 +344,7 @@ CT_DoLog EXTRA " host = ${CT_HOST}"
CT_DoLog EXTRA " target = ${CT_TARGET}"
set |egrep '^CT_.+=' |sort |CT_DoLog DEBUG
CT_EndStep
fi
# Include sub-scripts instead of calling them: that way, we do not have to
# export any variable, nor re-parse the configuration and functions files.
@ -322,7 +356,7 @@ CT_EndStep
. "${CT_TOP_DIR}/scripts/build/cc_${CT_CC}.sh"
. "${CT_TOP_DIR}/scripts/build/debug.sh"
# Now for the job by itself. Go have a coffee!
if [ -z "${CT_RESTART}" ]; then
CT_DoStep INFO "Retrieving needed toolchain components' tarballs"
do_kernel_get
do_binutils_get
@ -347,19 +381,40 @@ if [ "${CT_ONLY_DOWNLOAD}" != "y" ]; then
do_cc_extract
do_debug_extract
CT_EndStep
fi
fi
if [ "${CT_ONLY_EXTRACT}" != "y" ]; then
do_libc_check_config
do_kernel_check_config
do_kernel_headers
do_binutils
do_libc_headers
do_cc_core
do_libfloat
do_libc
do_cc
do_libc_finish
do_debug
# Now for the job by itself. Go have a coffee!
if [ "${CT_ONLY_DOWNLOAD}" != "y" -a "${CT_ONLY_EXTRACT}" != "y" ]; then
# Because of CT_RESTART, this becomes quite complex
[ -n "${CT_RESTART}" ] && do_it=0 || do_it=1
for step in libc_check_config \
kernel_check_config \
kernel_headers \
binutils \
libc_headers \
cc_core \
libfloat \
libc \
cc \
libc_finish \
debug \
; do
if [ ${do_it} -eq 0 ]; then
if [ "${CT_RESTART}" = "${step}" ]; then
CT_DoLoadState "${step}"
do_it=1
fi
else
CT_DoSaveState ${step}
fi
if [ ${do_it} -eq 1 ]; then
do_${step}
if [ "${CTDEBUG_CT_PAUSE_STEPS}" = "y" ]; then
CT_DoPause "Step \"${step}\" finished"
fi
fi
done
# Create the aliases to the target tools
if [ -n "${CT_TARGET_ALIAS}" ]; then
@ -381,7 +436,6 @@ if [ "${CT_ONLY_DOWNLOAD}" != "y" ]; then
rm -rf "${CT_DEBUG_INSTALL_DIR}/"{,usr/}{man,info}
fi
fi
fi
# OK, now we're done, set the toolchain read-only
# Don't log, the log file may become read-only any moment...

View File

@ -304,7 +304,7 @@ CT_GetFile() {
if [ -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" -a \
"${CT_FORCE_DOWNLOAD}" != "y" ]; then
CT_DoLog EXTRA "Copying \"${file}\" from local copy"
cp -v "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog DEBUG
cp -v "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL
return 0
fi
done
@ -460,3 +460,112 @@ CT_DoBuildTargetTriplet() {
esac
CT_TARGET="`${CT_TOP_DIR}/tools/config.sub ${CT_TARGET}`"
}
# This function does pause the build until the user strikes "Return"
# Usage: CT_DoPause [optional_message]
CT_DoPause() {
local foo
local message="${1:-Pausing for your pleasure}"
CT_DoLog INFO "${message}"
read -p "Press \"Enter\" to continue, or Ctrl-C to stop..." foo >&6
return 0
}
# This function saves the state of the toolchain to be able to restart
# at any one point
# Usage: CT_DoSaveState <next_step_name>
CT_DoSaveState() {
[ "${CT_DEBUG_CT_SAVE_STEPS}" = "y" ] || return 0
local state_name="$1"
local state_dir="${CT_STATE_DIR}/${state_name}"
CT_DoLog DEBUG "Saving state to restart at step \"${state_name}\"..."
rm -rf "${state_dir}"
mkdir -p "${state_dir}"
case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
y) tar_opt=czf; tar_ext=".tar.gz";;
*) tar_opt=cf; tar_ext=".tar";;
esac
CT_DoLog DEBUG " Saving environment and aliases"
# We must omit shell functions
# 'isgrep' is here because I don't seem to
# be able to remove the functions names.
set |awk '
BEGIN { _p = 1; }
$0~/^[^ ] ()/ { _p = 0; }
_p == 1
$0 == "}" { _p = 1; }
' |egrep -v '^[^ ]+ \(\)' >"${state_dir}/env.sh"
CT_DoLog DEBUG " Saving CT_CC_CORE_PREFIX_DIR=\"${CT_CC_CORE_PREFIX_DIR}\""
CT_Pushd "${CT_CC_CORE_PREFIX_DIR}"
tar ${tar_opt} "${state_dir}/cc_core_prefix_dir${tar_ext}" .
CT_Popd
CT_DoLog DEBUG " Saving CT_PREFIX_DIR=\"${CT_PREFIX_DIR}\""
CT_Pushd "${CT_PREFIX_DIR}"
tar ${tar_opt} "${state_dir}/prefix_dir${tar_ext}" .
CT_Popd
if [ "${CT_LOG_TO_FILE}" = "y" ]; then
CT_DoLog DEBUG " Saving log file"
exec >/dev/null
case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
y) gzip -3 -c "${CT_LOG_FILE}" >"${state_dir}/log.gz";;
*) cat "${CT_LOG_FILE}" >"${state_dir}/log";;
esac
exec >>"${CT_LOG_FILE}"
fi
}
# This functions restores a previously saved state
# Usage: CT_DoLoadState <state_name>
CT_DoLoadState(){
local state_name="$1"
local state_dir="${CT_STATE_DIR}/${state_name}"
# We need to do something special with the log file!
if [ "${CT_LOG_TO_FILE}" = "y" ]; then
exec >"${state_dir}/tail.log"
fi
CT_DoLog DEBUG "Restoring state at step \"${state_name}\"..."
case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
y) tar_opt=xzf; tar_ext=".tar.gz";;
*) tar_opt=cf; tar_ext=".tar";;
esac
CT_DoLog DEBUG " Removing previous build directories"
chmod -R u+rwX "${CT_PREFIX_DIR}" "${CT_CC_CORE_PREFIX_DIR}"
rm -rf "${CT_PREFIX_DIR}" "${CT_CC_CORE_PREFIX_DIR}"
mkdir -p "${CT_PREFIX_DIR}" "${CT_CC_CORE_PREFIX_DIR}"
CT_DoLog DEBUG " Restoring CT_PREFIX_DIR=\"${CT_PREFIX_DIR}\""
CT_Pushd "${CT_PREFIX_DIR}"
tar ${tar_opt} "${state_dir}/prefix_dir${tar_ext}"
CT_Popd
CT_DoLog DEBUG " Restoring CT_CC_CORE_PREFIX_DIR=\"${CT_CC_CORE_PREFIX_DIR}\""
CT_Pushd "${CT_CC_CORE_PREFIX_DIR}"
tar ${tar_opt} "${state_dir}/cc_core_prefix_dir${tar_ext}"
CT_Popd
# Restore the environment, discarding any error message
# (for example, read-only bash internals)
CT_DoLog DEBUG " Restoring environment"
. "${state_dir}/env.sh" >/dev/null 2>&1 || true
if [ "${CT_LOG_TO_FILE}" = "y" ]; then
CT_DoLog DEBUG " Restoring log file"
exec >/dev/null
case "${CT_DEBUG_CT_SAVE_STEPS_GZIP}" in
y) zcat "${state_dir}/log.gz" >"${CT_LOG_FILE}";;
*) cat "${state_dir}/log" >"${CT_LOG_FILE}";;
esac
cat "${state_dir}/tail.log" >>"${CT_LOG_FILE}"
exec >>"${CT_LOG_FILE}"
rm -f "${state_dir}/tail.log"
fi
}