tool/depot: improve handling of missing ports

* The extract tool determines and reports all missing ports at once.
* The extract tool automatically prepares all missing ports if PREPARE_PORTS=1.
* The missing_ports tool prints a list of missing ports for given archives.

Fixes #3353
This commit is contained in:
Martin Stein 2019-05-12 03:13:02 +02:00 committed by Christian Helmuth
parent 6d8d6b5552
commit 95ece89cf8
5 changed files with 310 additions and 68 deletions

View File

@ -48,6 +48,8 @@ define HELP_MESSAGE
archive version needs to be updated. The default archive version needs to be updated. The default
is to use the current date. is to use the current date.
PREPARE_PORTS=1 Prepare missing ports automatically.
endef endef
export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..)
@ -57,71 +59,9 @@ BIN_PKG_PATH_ELEMS := 4
include $(GENODE_DIR)/tool/depot/mk/front_end.inc include $(GENODE_DIR)/tool/depot/mk/front_end.inc
include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc
include $(GENODE_DIR)/tool/depot/mk/extract_pre_dependencies.inc
#
# Collect dependencies for all specified arguments
#
# The following accessor functions used by 'mk/dependencies.inc'. The
# information found in the 'archives' file of a package recipe has the
# placeholder '_' for the user. Only archives with this placeholder are
# considered. The '_user_pkg_archives' function transforms those archive paths
# into user-specific archive paths.
#
_file_in_depot = $(wildcard $(DEPOT_DIR)/$(call archive_user,$1)/src/$(call archive_recipe,$1)/$2)
_file_in_recipe = $(addsuffix /$2,$(call recipe_dir,src/$(call archive_recipe,$1)))
_file_in_depot_or_recipe = $(if $(call _file_in_depot,$1,$2),\
$(call _file_in_depot,$1,$2),\
$(call _file_in_recipe,$1,$2))
api_file = $(call _file_in_depot_or_recipe,$1,api)
used_apis_file = $(call _file_in_depot_or_recipe,$1,used_apis)
_pkg_archives_file = $(call recipe_dir,pkg/$(call archive_recipe,$1))/archives
_user_pkg_archives = $(patsubst _/%,$(call archive_user,$1)/%,\
$(call grep_archive_user,_,\
$(call file_content,$(call _pkg_archives_file,$1))))
pkg_src_archives = $(call grep_archive_type,src,$(call _user_pkg_archives,$1))
pkg_raw_archives = $(call grep_archive_type,raw,$(call _user_pkg_archives,$1))
pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _user_pkg_archives,$1))
include $(GENODE_DIR)/tool/depot/mk/dependencies.inc include $(GENODE_DIR)/tool/depot/mk/dependencies.inc
include $(GENODE_DIR)/tool/depot/mk/extract_post_dependencies.inc
#
# Obtain version information from recipes
#
# The 'archive_curr_version' function takes the archive type and name as
# arguments and returns the version identifier as present in the corresponding
# recipe. The nested foreach loop populates 'ARCHIVE_VERSION' with the version
# identifier for each archive.
#
# If an archive is given with a complete (versioned) name, we don't need to
# consult any recipe but only check if the corresponding archive exists within
# the depot. For binary archives, it suffices that the corresponding source
# archive is present.
#
$(foreach TYPE,api src raw pkg,\
$(foreach PATH,${ARCHIVES(${TYPE})},\
$(eval ARCHIVE_VERSION(${PATH}) := $(call archive_curr_version,$(PATH)))))
archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1)
ARCHIVES_WITH_NO_VERSION := $(sort \
$(foreach TYPE,api src raw pkg,\
$(foreach A,${ARCHIVES(${TYPE})},\
$(if $(call archive_exists_in_depot,$A),,\
$(if ${ARCHIVE_VERSION($A)},,$A)))))
checked_versions_defined:
ifneq ($(ARCHIVES_WITH_NO_VERSION),)
@echo "Error: incomplete or missing recipe ($(sort $(ARCHIVES_WITH_NO_VERSION)))"; false
endif
# #
# Generate makefile for archive-extraction stage # Generate makefile for archive-extraction stage
@ -140,7 +80,7 @@ wipe_existing_archives:
$(foreach A,${ARCHIVES(${TYPE})},\ $(foreach A,${ARCHIVES(${TYPE})},\
$(call versioned_archive,$A)))) $(call versioned_archive,$A))))
$(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized $(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized checked_ports_exist
$(VERBOSE)mkdir -p $(dir $@) $(VERBOSE)mkdir -p $(dir $@)
$(VERBOSE)( echo -e "all:\n"; \ $(VERBOSE)( echo -e "all:\n"; \
echo "TOOL_DIR := $(GENODE_DIR)/tool"; \ echo "TOOL_DIR := $(GENODE_DIR)/tool"; \
@ -177,8 +117,10 @@ $(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized
# Invoke sub make to process generated makefile # Invoke sub make to process generated makefile
# #
execute_generated_extract_mk_file: $(EXTRACT_MK_FILE) execute_generated_extract_mk_file: $(EXTRACT_MK_FILE)
$(VERBOSE)$(MAKE) $(if $(VERBOSE),--quiet) -f $(EXTRACT_MK_FILE) \ $(VERBOSE)$(MAKE) $(QUIET) \
-C $(DEPOT_DIR) VERBOSE=$(VERBOSE) \ -f $(EXTRACT_MK_FILE) \
-C $(DEPOT_DIR) \
VERBOSE=$(VERBOSE) \
UPDATE_VERSIONS=$(UPDATE_VERSIONS) UPDATE_VERSIONS=$(UPDATE_VERSIONS)
ifneq ($(FORCE),) ifneq ($(FORCE),)
@ -188,3 +130,43 @@ endif
$(MAKECMDGOALS): execute_generated_extract_mk_file $(MAKECMDGOALS): execute_generated_extract_mk_file
@true @true
#
# Return shell script that handles missing ports
#
# \param $1 sorted list of missing ports file
# \param $2 value of $(MAKE)
#
ifeq ($(PREPARE_PORTS),1)
# automatic prepare enabled: prepare missing ports
handle_missing_ports = $(if $1,$(VERBOSE)( \
echo -e ""; \
echo -e "Ports not prepared or outdated:"; \
echo -e " $(1)"; \
echo -e ""; \
echo -e "Preparing ports..."; \
echo -e ""; \
$2 $(QUIET) \
-f $(GENODE_DIR)/tool/ports/prepare_port \
$(1); ),)
else
# automatic prepare not enabled: generate error
handle_missing_ports = $(if $1,$(VERBOSE)( \
echo -e ""; \
echo -e "Error: Ports not prepared or outdated:"; \
echo -e " $(1)"; \
echo -e ""; \
echo -e "You can prepare respectively update them as follows:"; \
echo -e " $(GENODE_DIR)/tool/ports/prepare_port $(1)"; \
echo -e ""; \
false),)
endif
#
# Check whether all required ports exist and if not, abort or prepare them
#
checked_ports_exist: update_missing_ports_file
$(call handle_missing_ports,$(call sorted_file_content,$(MISSING_PORTS_FILE)),$(MAKE))

52
tool/depot/missing_ports Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/make -f
#
# \brief Print a list of missing ports for given API/source/raw archives
# \author Martin Stein
# \date 2019-05-15
#
define HELP_MESSAGE
Print a list of missing ports for given API/source/raw archives
usage:
$(firstword $(MAKEFILE_LIST)) <archive-path>...
The <archive-path> argument denotes the archive to process in the
form of a path. The first path element corresponds to the identity
of the archive creator, the second element corresponds to the type
of the archive, and the third element refers to the recipe of
the archive description.
E.g., the user 'alan' may have the following archives:
alan/api/libc - an API archive for the libc
alan/src/zlib - a source archive for the zlib library
The following arguments tweak the operation of the tool:
VERBOSE= Show individual operations.
-j<N> Enable the parallel processing of packages where
<N> denotes the level of parallelism.
endef
export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..)
# the missing-ports tool expects archive paths given without the version element
BIN_PKG_PATH_ELEMS := 4
include $(GENODE_DIR)/tool/depot/mk/front_end.inc
include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc
include $(GENODE_DIR)/tool/depot/mk/extract_pre_dependencies.inc
include $(GENODE_DIR)/tool/depot/mk/dependencies.inc
include $(GENODE_DIR)/tool/depot/mk/extract_post_dependencies.inc
$(MAKECMDGOALS): dump_missing_ports_file
$(VERBOSE)true
dump_missing_ports_file: update_missing_ports_file
$(VERBOSE)echo $(call sorted_file_content,$(MISSING_PORTS_FILE))

View File

@ -0,0 +1,57 @@
#
# \brief Environment for content.mk files when determining missing ports
# \author Martin Stein
# \date 2019-05-12
#
# GENODE_DIR - root directory of the Genode source tree
# CONTRIB_DIR - directory for 3rd-party code
# CONTENT_MK - content.mk file to process
# REP_DIR - repository directory of the content.mk file
# MISSING_PORTS_FILE - file to write the names of missing ports to
# VERBOSE - verbosity
#
#
# Functions for disabling and re-enabling evaluation of $(shell ...)
#
ORIGINAL_SHELL := $(SHELL)
enable_shell = $(eval SHELL:=$(ORIGINAL_SHELL))
disable_shell = $(eval SHELL:=true)
#
# Disable shell calls so the content.mk file will not evaluate something like
# $(shell find $(PORT_DIR) ...) while 'PORT_DIR' is empty because we have
# overridden the port_dir function.
#
$(disable_shell)
#
# If a port is missing, append its name to the missing ports file
#
_assert = $(if $1,$1,$(shell echo $2 >> $(MISSING_PORTS_FILE)))
#
# Utility to query the port directory for a given path to a port description.
#
# Example:
#
# $(call port_dir,$(GENODE_DIR)/repos/libports/ports/libpng)
#
_port_hash = $(shell cat $(call _assert,$(wildcard $1.hash),$(notdir $1)))
_port_dir = $(wildcard $(CONTRIB_DIR)/$(notdir $1)-$(call _port_hash,$1))
port_dir = $(call enable_shell)$(call _assert,$(call _port_dir,$1),$(notdir $1))$(call disable_shell)
#
# Prevent the evaluation of mirror_from_rep_dir in content.mk
#
mirror_from_rep_dir = $(error mirror_from_rep_dir called outside of target)
#
# Prevent the evaluation of the first target in the content.mk file
#
prevent_execution_of_content_targets:
#
# Include the content.mk file to evaluate all calls to the port_dir function
#
include $(CONTENT_MK)

View File

@ -0,0 +1,123 @@
#
# Obtain version information from recipes
#
# The 'archive_curr_version' function takes the archive type and name as
# arguments and returns the version identifier as present in the corresponding
# recipe. The nested foreach loop populates 'ARCHIVE_VERSION' with the version
# identifier for each archive.
#
# If an archive is given with a complete (versioned) name, we don't need to
# consult any recipe but only check if the corresponding archive exists within
# the depot. For binary archives, it suffices that the corresponding source
# archive is present.
#
$(foreach TYPE,api src raw pkg,\
$(foreach PATH,${ARCHIVES(${TYPE})},\
$(eval ARCHIVE_VERSION(${PATH}) := $(call archive_curr_version,$(PATH)))))
archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1)
ARCHIVES_WITH_NO_VERSION := $(sort \
$(foreach TYPE,api src raw pkg,\
$(foreach A,${ARCHIVES(${TYPE})},\
$(if $(call archive_exists_in_depot,$A),,\
$(if ${ARCHIVE_VERSION($A)},,$A)))))
checked_versions_defined:
ifneq ($(ARCHIVES_WITH_NO_VERSION),)
@echo "Error: incomplete or missing recipe ($(sort $(ARCHIVES_WITH_NO_VERSION)))"; false
endif
#
# Read content of a file as list, sort it and remove duplicates
#
# \param $1 absolute file path
#
sorted_file_content = $(if $(wildcard $1),$(shell cat $1 | sort -u),\
$(error Failed to read file $1))
#
# Absolute path to content.mk file for given archive
#
# \param $1 archive type, can be 'src', 'api' or 'raw'
# \param $2 api, src or raw archive name in the form 'genodelabs/api/base'
#
content_mk_file = $(addsuffix /content.mk,$(call recipe_dir,$1/$(call archive_recipe,$2)))
#
# Return a given string minus another given string
#
# \param $1 string that shall be returned minus the other one
# \param $2 string that shall be removed from the other one
#
remove_from_string=$(1:$2=)
#
# Repository directory for a given recipe name
#
# \param $1 recipe type in the form 'api'
# \param $2 recipe name in the form 'base'
#
rep_dir_of_recipe=$(call remove_from_string,$(call recipe_dir,$1/$2),/recipes/$1/$2)
#
# Repository directory for a given archive
#
# \param $1 archive type in the form 'api'
# \param $2 archive name in the form 'genodelabs/api/base'
#
rep_dir_of_archive=$(call rep_dir_of_recipe,$1,$(call archive_recipe,$2))
# path to temporary file that is used to buffer the names of missing ports
MISSING_PORTS_FILE := $(DEPOT_DIR)/var/missing_ports
# path to temporary make file that is created to fill the missing-ports file
GEN_MISSING_PORTS_MK := $(DEPOT_DIR)/var/gen_missing_ports.mk
# wether to invoke sub-makes with '--quiet'
QUIET = $(if $(VERBOSE),--quiet)
#
# Invoke sub-make to create or update missing-ports file
#
update_missing_ports_file: checked_versions_defined checked_no_uncategorized
$(VERBOSE)mkdir -p $(dir $(GEN_MISSING_PORTS_MK))
$(VERBOSE)( echo -e "all:\n"; \
echo -e "MAKE := $(MAKE)\n"; \
$(foreach TYPE,api src raw pkg,\
$(foreach A,${ARCHIVES(${TYPE})},\
target=$(call versioned_archive,$A); \
content_mk=$(call content_mk_file,$(TYPE),$A); \
rep_dir=$(call rep_dir_of_archive,$(TYPE),$A); \
echo "ARCHIVES(${TYPE}) += $$target"; \
echo "CONTENT_MK($$target) := $$content_mk"; \
echo "REP_DIR($$target) := $$rep_dir"; \
) ) \
echo -e ""; \
$(foreach A,${ARCHIVES(pkg)},\
$(foreach DEP,$(call pkg_pkg_archives,$A),\
echo -e "$(call versioned_archive,$A) :" \
"$(call versioned_archive,$(DEP))";)) \
echo -e ""; \
echo -e "\$${ARCHIVES(src)} : \$${ARCHIVES(api)}"; \
echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(api)}"; \
echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(src)}"; \
echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(raw)}"; \
echo -e "\nTARGETS := \$$(foreach T,api src raw,\$${ARCHIVES(\$$T)})"; \
echo -e "\nall: \$$(TARGETS)"; \
echo -e "\n\$$(TARGETS):"; \
echo -e "\t$(VERBOSE)\$$(MAKE) \\"; \
echo -e "\t $(QUIET) \\"; \
echo -e "\t -f $(GENODE_DIR)/tool/depot/mk/content_env_missing_ports.mk \\"; \
echo -e "\t GENODE_DIR=$(GENODE_DIR) \\"; \
echo -e "\t CONTRIB_DIR=$(GENODE_DIR)/contrib \\"; \
echo -e "\t CONTENT_MK=\$${CONTENT_MK(\$$@)} \\"; \
echo -e "\t REP_DIR=\$${REP_DIR(\$$@)} \\"; \
echo -e "\t MISSING_PORTS_FILE=$(MISSING_PORTS_FILE) \\"; \
echo -e "\t VERBOSE=$(VERBOSE)"; \
) > $(GEN_MISSING_PORTS_MK)
$(VERBOSE)mkdir -p $(dir $(MISSING_PORTS_FILE))
$(VERBOSE)rm -f $(MISSING_PORTS_FILE)
$(VERBOSE)touch $(MISSING_PORTS_FILE)
$(VERBOSE)$(MAKE) $(QUIET) -C $(DEPOT_DIR) -f $(GEN_MISSING_PORTS_MK);

View File

@ -0,0 +1,28 @@
#
# Collect dependencies for all specified arguments
#
# The following accessor functions used by 'mk/dependencies.inc'. The
# information found in the 'archives' file of a package recipe has the
# placeholder '_' for the user. Only archives with this placeholder are
# considered. The '_user_pkg_archives' function transforms those archive paths
# into user-specific archive paths.
#
_file_in_depot = $(wildcard $(DEPOT_DIR)/$(call archive_user,$1)/src/$(call archive_recipe,$1)/$2)
_file_in_recipe = $(addsuffix /$2,$(call recipe_dir,src/$(call archive_recipe,$1)))
_file_in_depot_or_recipe = $(if $(call _file_in_depot,$1,$2),\
$(call _file_in_depot,$1,$2),\
$(call _file_in_recipe,$1,$2))
api_file = $(call _file_in_depot_or_recipe,$1,api)
used_apis_file = $(call _file_in_depot_or_recipe,$1,used_apis)
_pkg_archives_file = $(call recipe_dir,pkg/$(call archive_recipe,$1))/archives
_user_pkg_archives = $(patsubst _/%,$(call archive_user,$1)/%,\
$(call grep_archive_user,_,\
$(call file_content,$(call _pkg_archives_file,$1))))
pkg_src_archives = $(call grep_archive_type,src,$(call _user_pkg_archives,$1))
pkg_raw_archives = $(call grep_archive_type,raw,$(call _user_pkg_archives,$1))
pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _user_pkg_archives,$1))