Build-system support for ported 3rd-party code

The new 'select_from_ports' function allows a target description file to
query the path to an installed port. All ports are stored in a central
location specified as CONTRIB_DIR. By default, CONTRIB_DIR is defined
as '<genode-dir>/contrib'. Ports of 3rd-party source code are managed
using the tools at '<genode-dir>/tool/ports/'.

Issue #1082

repos/base/mk/dep_lib.mk

@@ -17,9 +17,9 @@
 #
 
 #
-# Utility for selecting files from the list of repositories
+# Include common utility functions
 #
-select_from_repositories = $(firstword $(foreach REP,$(REPOSITORIES),$(wildcard $(REP)/$(1))))
+include $(BASE_DIR)/mk/util.inc
 
 #
 # Generate dependencies only for those libs that are

repos/base/mk/dep_prg.mk

@@ -4,9 +4,9 @@
 all:
 
 #
-# Utility for selecting files from the list of repositories
+# Include common utility functions
 #
-select_from_repositories = $(firstword $(foreach REP,$(REPOSITORIES),$(wildcard $(REP)/$(1))))
+include $(BASE_DIR)/mk/util.inc
 
 #
 # Include target build instructions to aquire library dependecies

repos/base/mk/generic.mk

@@ -30,6 +30,15 @@ $(wildcard $(OBJECTS)): $(filter-out $(LIB_PROGRESS_LOG),$(MAKEFILE_LIST))
 
 INCLUDES := $(addprefix -I,$(wildcard $(ALL_INC_DIR)))
 
+#
+# If one of the 3rd-party ports used by the target changed, we need to rebuild
+# all object files because they may include headers from the 3rd-party port.
+#
+# The 'PORT_HASH_FILES' variable is populated as side effect of calling the
+# 'select_from_ports' function.
+#
+$(OBJECTS): $(PORT_HASH_FILES)
+
 #
 # Include dependency files for the corresponding object files except
 # when cleaning

repos/base/mk/lib.mk

@@ -13,6 +13,7 @@
 ##   INSTALL_DIR      - program target build directory
 ##   DEPS             - library dependencies
 ##   REP_DIR          - repository where the library resides
+##   CONTRIB_DIR      - location of ported 3rd-party source codes
 ##
 
 include $(BASE_DIR)/mk/base-libs.mk
@@ -23,10 +24,9 @@ include $(BASE_DIR)/mk/base-libs.mk
 all:
 
 #
-# Function that searches for files in all
-# repositories and returns the first match.
+# Include common utility functions
 #
-select_from_repositories = $(firstword $(foreach REP,$(REPOSITORIES),$(wildcard $(REP)/$(1))))
+include $(BASE_DIR)/mk/util.inc
 
 #
 # Include specifics, for example platform, kernel-api etc.
@@ -36,7 +36,7 @@ include $(SPEC_FILES)
 #
 # Include library build instructions
 #
-# We set the 'called_from_lib_mk' variable to allow the library decription file
+# We set the 'called_from_lib_mk' variable to allow the library description file
 # to respond to the build pass.
 #
 BACKUP_INC_DIR := $(INC_DIR)

repos/base/mk/prg.mk

@@ -20,9 +20,9 @@
 all:
 
 #
-# Function that searches for files in all repositories and returns the first match
+# Include common utility functions
 #
-select_from_repositories = $(firstword $(foreach REP,$(REPOSITORIES),$(wildcard $(REP)/$(1))))
+include $(BASE_DIR)/mk/util.inc
 
 #
 # Include target build instructions

repos/base/mk/util.inc

@@ -0,0 +1,45 @@
+#
+# Utility for selecting files from the list of repositories
+#
+select_from_repositories = $(firstword $(foreach REP,$(REPOSITORIES),$(wildcard $(REP)/$(1))))
+
+#
+# Check presence of argument $1. Back out with error message $2 if not defined.
+#
+_assert = $(if $1,$1,$(error Error: $2))
+
+#
+# Append value $1 to variable named $2 and return value $1
+#
+# We must not specify an '=' here. Even though the make documentation states
+# that the omission of '=' should be equivalent to '=', the behaviour is not
+# the same.
+#
+define _capture
+$(eval $2 += $1)
+$1
+endef
+
+#
+# Lookup port directory by a given port name
+#
+# The location of the port directory is determined by the hash value stored
+# in the corresponding 'ports/<port>.hash' file. First we have to find the
+# hash file in one of the REPOSITORIES. Once we found the hash file, we use
+# its contained hash number to construct the path to the corresponding
+# subdirectory within CONTRIB_DIR. Finally, we check if the path exists.
+#
+# When reading the hash file in the '_hash_of_port' function, we feed stdin
+# to 'cat' to prevent 'cat' from blocking if the hash file is missing.
+#
+# As a side effect of calling 'select_from_ports' we log the used hash file
+# in the 'PORT_HASH_FILES' variable. This enables us incorporate the hash file
+# as dependency for all object files.
+#
+_lookup_port_hash_file  = $(wildcard $(addsuffix /ports/$1.hash,$(REPOSITORIES)))
+_capture_port_hash_file = $(call _capture,$(call _lookup_port_hash_file,$1),PORT_HASH_FILES)
+_hash_of_port           = $(shell echo | cat $(call _capture_port_hash_file,$1))
+_port_dir               = $(wildcard $(CONTRIB_DIR)/$1-$(call _hash_of_port,$1))
+_checked_port_dir       = $(call _assert,$(call _port_dir,$1),$1 is not prepared or outdated)
+
+select_from_ports = $(call _checked_port_dir,$1)

tool/builddir/build.mk

@@ -30,6 +30,8 @@
 #                 Normally, the libcache is located at 'var/libcache' and
 #                 there is no need to change it.
 #
+# CONTRIB_DIR   - location of ported 3rd-party source codes
+#
 
 ##
 ## Define global configuration variables
@@ -49,6 +51,7 @@ export LIB_CACHE_DIR    ?= $(BUILD_BASE_DIR)/var/libcache
 export LIB_PROGRESS_LOG ?= $(BUILD_BASE_DIR)/progress.log
 export LIB_DEP_FILE     ?= var/libdeps
 export ECHO             ?= echo -e
+export CONTRIB_DIR
 
 #
 # Convert user-defined directories to absolute directories

tool/create_builddir

@@ -61,6 +61,8 @@ GENODE_DIR ?= $(realpath $(dir $(MAKEFILE_LIST))/..)
 #
 BUILD_DIR ?= $(GENODE_DIR)/build/$(PLATFORM)
 
+SHELL := bash
+
 ifneq ($(wildcard $(BUILD_DIR)),)
 $(PLATFORM):: build_dir_already_exists
 build_dir_already_exists:
@@ -75,6 +77,18 @@ endif
 #
 GENODE_ABS_DIR := $(realpath $(shell echo $(GENODE_DIR)))
 
+#
+# Define absolute path to the contrib directory as written to the
+# 'etc/build.conf' file. We use 'abs_path' instead of 'realpath' because the
+# contrib directory may not exist when the build directory is created. In this
+# case, 'realpath' would return an empty string.
+#
+ifeq ($(CONTRIB_DIR),)
+CONTRIB_ABS_DIR := $$(GENODE_DIR)/contrib
+else
+CONTRIB_ABS_DIR := $(abspath $(shell echo $(CONTRIB_DIR)))
+endif
+
 #
 # Generic build-directory preparation rules
 #
@@ -87,6 +101,7 @@ $(BUILD_DIR)/etc:
 $(BUILD_DIR)/etc/build.conf:: $(BUILD_CONF_PLATFORM)
 	@echo "GENODE_DIR    = $(GENODE_ABS_DIR)" > $@
 	@echo 'BASE_DIR      = $$(GENODE_DIR)/repos/base' >> $@
+	@echo 'CONTRIB_DIR   = $(CONTRIB_ABS_DIR)' >> $@
 
 #
 # Supply -no-kvm argument to Qemu for kernels that are incompatible with KVM