diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..ab5ee88d9ef0dfc45d89607e3330d98d3f1e3e63 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.makefile gitlab-language=make diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d0afc9e4f1d1ba7e78e05edb9e4732bad3ce78f..29b6724c8df5fb06fdf49c9d39e214ba3662dcb8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/ambv/black - rev: 21.12b0 + rev: 22.3.0 hooks: - id: black - repo: https://gitlab.com/PyCQA/flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7148863438e1252d4deef81bd85d35cb4f422394..26ba041c00c87cb0b2132f58cb54a367dd4d3a32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### New Features +* Recursive dependency for headers at build-time. This is a more major change that involves: + * Module dependecies are now fetched from `CONFIG_MODULE` in the sense that `X_DEP_VERSION` is parsed as a + dependency on the module `x` + * It is no longer necessary to specify `REQUIRED += ...` nor `x_VERSION = $(X_DEP_VERSION)` within the module makefile ### Bugfixes @@ -20,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Autocompletion for `iocsh.bash` has been added * Removed `iocsh_gdb.bash` and `iocsh_nice.bash`, both of whose functionality can be called via `iocsh.bash -dg` and `iocsh.bash -n`, respectively. * Patch file location has been change from `patch/Site/$VERSION-description.p0.patch` to `patch/Site/$VERSION/description.p0.patch` -* Require will automatically build `.template` and `.substitutions` files into the common build directory instead of into the source Db path +* Require will automatically build `.template` and `.substitutions` files into the common build directory instead of into the source Db path * Rudimentary testing has been added: * * Tests that the correct version is loaded * * Tests that elementary patching/building works as expected diff --git a/configure/E3/CONFIG_EXPORT b/configure/E3/CONFIG_EXPORT index c3cfd358889fa1cac42808f1e055904eeccc5382..159a88ee8fe08de35f26519edad7ca48148b6661 100644 --- a/configure/E3/CONFIG_EXPORT +++ b/configure/E3/CONFIG_EXPORT @@ -13,7 +13,7 @@ EXPORT_VARS+=E3_SITEMODS_PATH EXPORT_VARS+=$(filter E3_REQUIRE_%,$(.VARIABLES)) EXPORT_VARS+=QUIET -EXPORT_VARS+=$(filter %_DEP_VERSION,$(.VARIABLES)) +EXPORT_VARS+=$(foreach v,$(.VARIABLES),$(if $(findstring _DEP_VERSION,$v),$v)) EXPORT_VARS+=$(filter WITH_%,$(.VARIABLES)) EXPORT_VARS+=$(filter %_EXTERNAL,$(.VARIABLES)) diff --git a/require-ess/tools/driver.makefile b/require-ess/tools/driver.makefile index 346d825e31ed33f458d8cefd2ab1776a5f21e826..d9c499d6c226e04fe874094dd6cf890125fcead3 100644 --- a/require-ess/tools/driver.makefile +++ b/require-ess/tools/driver.makefile @@ -146,6 +146,23 @@ define FETCH_BUILD_NUMBER $(shell $(MAKEHOME)/build_number.sh $(1) $(2) $($(2)_VERSION)) endef +# Functions used for recursive dependency fetching. These are modified from https://github.com/markpiffer/gmtt.git +space := $(strip) $(strip)# +comma := ,# +list2param = $(subst $(space),$(comma),$(strip $1)) +exec = $(eval -exec=$1)$(eval -exec:=$$(call -exec,$(call list2param,$2)))$(-exec) +while = $(if $(call exec,$1),$(eval $2)$(call while,$1,$2,$3),$(eval $3)) + +# This simply assures that we fetch all of the rest of the table when we run $(wordlist n,$(max_int_size),$(table_data)) +max_int_size := 2147483647 + +# Used to select from table-like data; syntax is SELECT what FROM where WHEN condition. This has been tailored to parse .dep files. +select = $(strip $(call -select,$(strip $2),$1,$3)) +-select = $(if $1,$(if $(call exec,$3,$(call list2param,$(wordlist 1,2,$1))), $(word $2,$1))$(call -select,$(wordlist 3,$(max_int_size),$1),$2,$3)) + +str-eq = $(if $(subst x$1,,x$2),,t) +# End of functions from https://github.com/markpiffer/gmtt.git + ifndef EPICSVERSION ## RUN 1 # In source directory @@ -423,6 +440,52 @@ ifeq ($(filter O.%,$(notdir ${CURDIR})),) ## RUN 3 # Target architecture defined. # Still in source directory, third run. + +# Add sources for specific epics types or architectures. +ARCH_PARTS = ${T_A} $(subst -, ,${T_A}) ${OS_CLASS} +VAR_EXTENSIONS = ${EPICSVERSION} ${ARCH_PARTS} ${ARCH_PARTS:%=${EPICSVERSION}_%} +export VAR_EXTENSIONS + +MODULES := +PROCESSED_MODULES := +REQ := +INSTALLED_MODULES := $(sort $(notdir $(wildcard $(E3_SITEMODS_PATH)/* $(EPICS_MODULES)/*))) + +# Converts all of the X_DEP_VERSIONs to x_VERSION and records them +define fetch_module_versions + lm := $$(shell echo $1 | tr '[:upper:]' '[:lower:]') + ifneq ($$(strip $$(filter $(INSTALLED_MODULES),$$(lm))),) + $$(lm)_VERSION := $($1_DEP_VERSION$2) + MODULES += $$(lm) + REQ += $$(lm) + else + $$(warning Invalid dependency "$1_DEP_VERSION$2"; pruning) + endif +endef +$(foreach m,$(patsubst %_DEP_VERSION,%,$(filter %_DEP_VERSION,$(.VARIABLES))),$(eval $(call fetch_module_versions,$m))) +$(foreach x,$(VAR_EXTENSIONS),\ + $(foreach m,$(patsubst %_DEP_VERSION_$(x),%,$(filter %_DEP_VERSION_$(x),$(.VARIABLES))),$(eval $(call fetch_module_versions,$m,_$(x))))\ +) +export REQ + +# Fetches the data from .dep files to be parsed by the above +define fetch_deps +$(shell cat $(lastword $(wildcard $(addsuffix /$1/$($1_VERSION)/lib/$(T_A)/$1.dep,$(E3_SITEMODS_PATH) $(EPICS_MODULES)))) | sed '1d') +endef + +# Used to recurse through versions: recursively fetches all of the dependencies from the given module +define update_dep_versions + m := $$(firstword $$(MODULES)) + PROCESSED_MODULES += $$m + $$m_TBL := $$(call fetch_deps,$$m) + $$m_DEPS := $$(call select,1,$$($$m_TBL),1) + MODULES := $$(filter-out $$(PROCESSED_MODULES),$$(MODULES) $$($$m_DEPS)) + $$(foreach mm,$$($$m_DEPS),$$(eval $$(mm)_VERSION := $$(call select,2,$$($$m_TBL),$$$$(call str-eq,$$$$1,$$(mm))))) +endef +$(call while,$$(MODULES),$(update_dep_versions)) + +$(foreach m,$(PROCESSED_MODULES),$(eval export $m_VERSION)) + debug:: @echo "===================== Pass 3: T_A = $(T_A) =====================" @echo "BINS = $(BINS)" @@ -463,14 +526,6 @@ install build debug:: O.${EPICSVERSION}_${T_A} endif -# Add sources for specific epics types or architectures. -ARCH_PARTS = ${T_A} $(subst -, ,${T_A}) ${OS_CLASS} -VAR_EXTENSIONS = ${EPICSVERSION} ${ARCH_PARTS} ${ARCH_PARTS:%=${EPICSVERSION}_%} -export VAR_EXTENSIONS - -REQ = ${REQUIRED} $(foreach x, ${VAR_EXTENSIONS}, ${REQUIRED_$x}) -export REQ - SRCS += $(foreach x, ${VAR_EXTENSIONS}, ${SOURCES_$x}) USR_LIBOBJS += ${LIBOBJS} $(foreach x,${VAR_EXTENSIONS},${LIBOBJS_$x}) export USR_LIBOBJS diff --git a/tests/conftest.py b/tests/conftest.py index b78ea835ab16e17bd9bfe4b1a5795da0cc4c35cc..1f9a531387023a28ee1af2eb6023151f14d56c44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,9 @@ class Wrapper: self.config_dir = self.path / "configure" self.config_dir.mkdir() + self.config_module = self.path / "CONFIG_MODULE" + self.config_module.touch() + self.makefile = self.path / f"{name}.Makefile" makefile_contents = f""" @@ -34,6 +37,8 @@ E3_MODULE_VERSION?=0.0.0+0 E3_MODULE_SRC_PATH:={module_path} E3_MODULE_MAKEFILE:={name}.Makefile +include $(TOP)/CONFIG_MODULE + include $(REQUIRE_CONFIG)/CONFIG include $(REQUIRE_CONFIG)/RULES_SITEMODS """ @@ -62,6 +67,10 @@ include $(E3_REQUIRE_TOOLS)/driver.makefile for var, value in config_vars.items(): f.write(f"{var} = {value}\n") + def add_var_to_config_module(self, makefile_var: str, value: str, modifier="+"): + with open(self.config_module, "a") as f: + f.write(f"{makefile_var} {modifier}= {value}\n") + def add_var_to_makefile(self, makefile_var: str, value: str, modifier="+"): with open(self.makefile, "a") as f: f.write(f"{makefile_var} {modifier}= {value}\n") diff --git a/tests/test_build.py b/tests/test_build.py index e66ed7ca869ae3aa5e80037eb4377afd1eca113a..6670f6c7d88bd3e412cce484cee0b670596bc8ee 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -88,11 +88,11 @@ def test_missing_source_file(wrapper): def test_missing_requirement(wrapper): - wrapper.add_var_to_makefile("REQUIRED", "foo") + wrapper.add_var_to_config_module("FOO_DEP_VERSION", "bar") rc, _, errs = wrapper.run_make("build") - assert rc == 2 - assert "REQUIRED module 'foo' version '' does not exist" in errs + assert rc == 0 + assert 'Invalid dependency "FOO_DEP_VERSION"; pruning' in errs def test_header_install_location(wrapper): @@ -125,9 +125,8 @@ def test_updated_dependencies(wrappers): old_version = "0.0.0+0" - wrapper_main.add_var_to_makefile("REQUIRED", wrapper_dep.name) - wrapper_main.add_var_to_makefile( - f"{wrapper_dep.name}_VERSION", old_version, modifier="" + wrapper_main.add_var_to_config_module( + f"{wrapper_dep.name}_DEP_VERSION", old_version, modifier="" ) rc, *_ = wrapper_dep.run_make( @@ -149,8 +148,8 @@ def test_updated_dependencies(wrappers): ) assert rc == 0 - wrapper_main.add_var_to_makefile( - f"{wrapper_dep.name}_VERSION", new_version, modifier="" + wrapper_main.add_var_to_config_module( + f"{wrapper_dep.name}_DEP_VERSION", new_version, modifier="" ) rc, *_ = wrapper_main.run_make("cellinstall", module_version=new_version) @@ -161,3 +160,107 @@ def test_updated_dependencies(wrappers): ) assert rc == 0 assert f"Loaded {wrapper_dep.name} version {new_version}" in outs + + +def test_automated_dependency(wrappers): + wrapper_a = wrappers.get() + wrapper_b = wrappers.get() + + cell_path = wrapper_a.path / "cellMods" + + module_version = "0.0.0+0" + + wrapper_a.add_var_to_config_module(f"{wrapper_b.name}_DEP_VERSION", module_version) + + rc, *_ = wrapper_b.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + rc, *_ = wrapper_a.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + + for dep_file in (cell_path / wrapper_a.name).rglob("*.dep"): + with open(dep_file, "r") as f: + contents = f.readlines() + + assert len(contents) == 2 + assert contents[0].strip() == "# Generated file. Do not edit." + assert f"{wrapper_b.name} {module_version}" == contents[1] + + +def test_architecture_dependent_dependency(wrappers): + wrapper_a = wrappers.get() + wrapper_b = wrappers.get() + wrapper_c = wrappers.get() + + cell_path = wrapper_a.path / "cellMods" + + module_version = "0.0.0+0" + + wrapper_a.add_var_to_config_module( + f"{wrapper_b.name}_DEP_VERSION_linux", module_version + ) + wrapper_a.add_var_to_config_module( + f"{wrapper_c.name}_DEP_VERSION_not_an_arch", module_version + ) + + rc, *_ = wrapper_c.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + rc, *_ = wrapper_b.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + rc, *_ = wrapper_a.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + + rc, outs, _ = run_ioc_get_output( + wrapper_a.name, module_version, wrapper_a.path / "cellMods" + ) + assert rc == 0 + assert f"Loaded {wrapper_b.name} version {module_version}" in outs + assert f"Loaded {wrapper_c.name} version {module_version}" not in outs + + +def test_recursive_header_include(wrappers): + wrapper_a = wrappers.get() + wrapper_b = wrappers.get() + wrapper_c = wrappers.get() + + cell_path = wrapper_a.path / "cellMods" + + module_version = "0.0.0+0" + + wrapper_b.add_var_to_config_module(f"{wrapper_c.name}_DEP_VERSION", module_version) + wrapper_a.add_var_to_config_module(f"{wrapper_b.name}_DEP_VERSION", module_version) + + wrapper_c.add_var_to_makefile("HEADERS", f"{wrapper_c.name}.h") + (wrapper_c.module_dir / f"{wrapper_c.name}.h").touch() + + wrapper_a.add_var_to_makefile("SOURCES", f"{wrapper_a.name}.c") + with open(wrapper_a.module_dir / f"{wrapper_a.name}.c", "w") as f: + f.write(f'#include "{wrapper_c.name}.h"') + + rc, *_ = wrapper_c.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + rc, *_ = wrapper_b.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + rc, *_ = wrapper_a.run_make( + "cellinstall", module_version=module_version, cell_path=cell_path + ) + assert rc == 0 + + rc, outs, _ = run_ioc_get_output( + wrapper_a.name, module_version, wrapper_a.path / "cellMods" + ) + assert rc == 0 + assert f"Loaded {wrapper_c.name} version {module_version}" in outs