From cb94fb27590e24b15e779edd06d7ba01c80c8064 Mon Sep 17 00:00:00 2001
From: Simon Rose <simon.rose@ess.eu>
Date: Thu, 28 Oct 2021 15:11:16 +0200
Subject: [PATCH] Replace expandDBD.tcl with python script
The purpose of this is to remove the dependency on tclx, a non-standard
package.
---
CHANGELOG.md | 8 +++
configure/module/RULES_REQUIRE | 2 +-
require-ess/tools/driver.makefile | 4 +-
require-ess/tools/expandDBD | 103 ++++++++++++++++++++++++++
require-ess/tools/expandDBD.tcl | 115 ------------------------------
tests/test_expand_dbd.py | 56 +++++++++++----
6 files changed, 157 insertions(+), 131 deletions(-)
create mode 100755 require-ess/tools/expandDBD
delete mode 100755 require-ess/tools/expandDBD.tcl
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e5508b50..3581d885 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### New Features
+
+### Bugfixes
+
+### Other changes
+
+* Replaced `tclx` script to expand .dbd files with a python script
+
## [5.0.0]
### New Features
diff --git a/configure/module/RULES_REQUIRE b/configure/module/RULES_REQUIRE
index 2370898e..22cd10d5 100644
--- a/configure/module/RULES_REQUIRE
+++ b/configure/module/RULES_REQUIRE
@@ -23,7 +23,7 @@ endif
install: requireconf
requireconf: e3-site-path e3-require-path
- $(QUIET) install -m 755 $(wildcard $(E3_MODULE_SRC_PATH)/tools/*.tcl) $(E3_REQUIRE_TOOLS)/
+ $(QUIET) install -m 755 $(wildcard $(E3_MODULE_SRC_PATH)/tools/expandDBD) $(E3_REQUIRE_TOOLS)/
$(QUIET) install -m 644 $(E3_MODULE_SRC_PATH)/tools/driver.makefile $(E3_REQUIRE_TOOLS)/
$(QUIET) install -m 755 $(E3_MODULE_SRC_PATH)/tools/revision_number $(E3_REQUIRE_TOOLS)/
$(QUIET) install -m 755 $(E3_SHELL_FILES) $(E3_REQUIRE_BIN)/
diff --git a/require-ess/tools/driver.makefile b/require-ess/tools/driver.makefile
index ceab7598..bf7c6a6c 100644
--- a/require-ess/tools/driver.makefile
+++ b/require-ess/tools/driver.makefile
@@ -216,7 +216,7 @@ export DBD_SRCS
#record dbd files given in DBDS
RECORDS1 = $(patsubst %Record.dbd, %, $(filter-out dev%, $(filter %Record.dbd, $(notdir ${DBD_SRCS}))))
#record dbd files included by files given in DBDS
-RECORDS2 = $(filter-out dev%, $(shell ${MAKEHOME}/expandDBD.tcl -r $(addprefix -I, $(sort $(dir ${DBD_SRCS}))) $(realpath ${DBDS})))
+RECORDS2 = $(filter-out dev%, $(shell ${MAKEHOME}/expandDBD -r $(addprefix -I, $(sort $(dir ${DBD_SRCS}))) $(realpath ${DBDS}) 2> /dev/null))
RECORDS = $(sort ${RECORDS1} ${RECORDS2})
export RECORDS
@@ -610,7 +610,7 @@ MODULEINFOS:
# because it has too strict checks to be used for a loadable module.
${MODULEDBD}: ${DBDFILES}
@echo "Expanding $@"
- ${MAKEHOME}expandDBD.tcl -$(basename ${EPICSVERSION}) ${DBDEXPANDPATH} $^ > $@
+ ${MAKEHOME}expandDBD ${DBDEXPANDPATH} $^ > $@
# Install everything.
INSTALL_LIBS = ${MODULELIB:%=${INSTALL_LIB}/%}
diff --git a/require-ess/tools/expandDBD b/require-ess/tools/expandDBD
new file mode 100755
index 00000000..9e894f5b
--- /dev/null
+++ b/require-ess/tools/expandDBD
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import argparse
+import os
+import re
+import sys
+
+SCANNED_FILES = set()
+RECORD_REGEX = re.compile(r"include[ \t]*\"?((.*)Record.dbd)\"?")
+INCLUDE_REGEX = re.compile(r"include[ \t]+\"?([^\"]*)\"?")
+BLANK_LINE_REGEX = re.compile(r"^[ \t]*(#|%|$)")
+
+
+def find_dbd_file(filename, include_paths):
+ """Search for a given .dbd file among the include paths."""
+ for include_path in include_paths:
+ potential_path = os.path.join(include_path, filename)
+ if os.path.exists(potential_path):
+ return potential_path
+ return None
+
+
+def open_dbd_file(current_file, filename, includes=None):
+ """Open a .dbd file and return the lines."""
+ if not includes:
+ includes = ["."]
+
+ basename = os.path.basename(filename)
+ if basename in SCANNED_FILES:
+ print(
+ "Info: skipping duplicate file {basename} included from {current_file}".format(
+ basename=basename, current_file=current_file
+ ),
+ file=sys.stderr,
+ )
+ return []
+ dbd_file = find_dbd_file(basename, includes)
+ if dbd_file is None:
+ print("File '{basename}' not found".format(basename=basename), file=sys.stderr)
+ sys.exit(1)
+ SCANNED_FILES.add(basename)
+ with open(dbd_file, "r") as f:
+ return [line.strip() for line in f.readlines()]
+
+
+def expand_dbd_file(current_file, dbdlines, includes):
+ """Print to stdout a fully-expanded .dbd file."""
+ for line in dbdlines:
+ if BLANK_LINE_REGEX.match(line):
+ continue
+ m = INCLUDE_REGEX.search(line)
+ if m:
+ lines = open_dbd_file(current_file, m.group(1), includes)
+ expand_dbd_file(m.group(1), lines, includes)
+ else:
+ print(line)
+
+
+def parse_record_dbd_file(current_file, dbdlines, includes):
+ """Recursively search through .dbd files for xRecord.dbd, and print the record names."""
+ for line in dbdlines:
+ if BLANK_LINE_REGEX.match(line):
+ continue
+ m = RECORD_REGEX.search(line)
+ if m:
+ if find_dbd_file(m.group(1), includes):
+ print(m.group(2))
+ else:
+ print(
+ "File '{group}' not found".format(group=m.group(1)), file=sys.stderr
+ )
+ sys.exit(1)
+ continue
+ m = INCLUDE_REGEX.search(line)
+ if m:
+ lines = open_dbd_file(current_file, m.group(1), includes)
+ parse_record_dbd_file(m.group(1), lines, includes)
+
+
+def handle_dbd_file(dbdfiles, include, record_types):
+ """Process the .dbd files."""
+ for dbdfile in dbdfiles:
+ lines = open_dbd_file("command line", dbdfile, include)
+ if record_types:
+ parse_record_dbd_file(dbdfile, lines, include)
+ else:
+ expand_dbd_file(dbdfile, lines, include)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-I", "--include", action="append")
+ parser.add_argument("-r", "--record-types", action="store_true")
+ parser.add_argument("dbdfiles", nargs="+")
+
+ args = parser.parse_args()
+
+ handle_dbd_file(**vars(args))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/require-ess/tools/expandDBD.tcl b/require-ess/tools/expandDBD.tcl
deleted file mode 100755
index 3cd4bc78..00000000
--- a/require-ess/tools/expandDBD.tcl
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/env tclsh
-
-package require Tclx
-
-set global_context [scancontext create]
-
-set epicsversion 3.14
-set quiet 0
-set recordtypes 0
-set seachpath {}
-set filesDone {}
-
-while {[llength $argv]} {
- switch -glob -- [lindex $argv 0] {
- "-[0-9]*" { set epicsversion [string range [lindex $argv 0] 1 end]}
- "-q" { set quiet 1 }
- "-r" { set recordtypes 1; set quiet 1 }
- "-I" { lappend seachpath [lindex $argv 1]; set argv [lreplace $argv 0 1]; continue }
- "-I*" { lappend seachpath [string range [lindex $argv 0] 2 end] }
- "--" { set argv [lreplace $argv 0 0]; break }
- "-*" { puts stderr "Warning: Unknown option [lindex $argv 0] ignored" }
- default { break }
- }
- set argv [lreplace $argv 0 0]
-}
-
-proc opendbd {name} {
- global seachpath
- foreach dir $seachpath {
- if ![catch {
- set file [open [file join $dir $name]]
- }] {
- return $file
- }
- }
- return -code error "file $name not found"
-}
-
-scanmatch $global_context {^[ \t]*(#|%|$)} {
- continue
-}
-
-if {$recordtypes} {
- scanmatch $global_context {include[ \t]+"?((.*)Record.dbd)"?} {
- if ![catch {
- close [opendbd $matchInfo(submatch0)]
- }] {
- puts $matchInfo(submatch1)
- }
- continue
-}
-
-} else {
-
- scanmatch $global_context {(registrar|variable|function)[ \t]*\([ \t]*"?([a-zA-Z0-9_]+)"?[ \t]*\)} {
- global epicsversion
- if {$epicsversion > 3.13} {puts $matchInfo(submatch0)($matchInfo(submatch1))}
- }
- scanmatch $global_context {variable[ \t]*\([ \t]*"?([a-zA-Z0-9_]+)"?[ \t]*,[ \t]*"?([a-zA-Z0-9_]+)"?[ \t]*\)} {
- global epicsversion
- if {$epicsversion > 3.13} {puts variable($matchInfo(submatch0),$matchInfo(submatch1))}
- }
-
- scanmatch $global_context {
- puts $matchInfo(line)
- }
-}
-
-scanmatch $global_context {include[ \t]+"?([^"]*)"?} {
- global seachpath
- global FileName
- global quiet
- if [catch {
- includeFile $global_context $matchInfo(submatch0)
- } msg] {
- if {!$quiet} {
- puts stderr "ERROR: $msg in path \"$seachpath\" called from $FileName($matchInfo(handle)) line $matchInfo(linenum)"
- exit 1
- }
- }
- continue
-}
-
-proc includeFile {context filename} {
- global global_context FileName filesDone matchInfo quiet
- set basename [file tail $filename]
- if {[lsearch $filesDone $basename ] != -1} {
- if {!$quiet} {
- puts stderr "Info: skipping duplicate file $basename included from $FileName($matchInfo(handle))"
- }
- return
- }
- if {$filename != "dbCommon.dbd"} { lappend filesDone [file tail $filename] }
- set file [opendbd $filename]
- set FileName($file) $filename
- #puts "#include $filename from $FileName($matchInfo(handle))"
- scanfile $context $file
- close $file
-}
-
-foreach filename $argv {
- global filesDone quiet
- set basename [file tail $filename]
- if {[lsearch $filesDone $basename] != -1} {
- if {!$quiet} {
- puts stderr "Info: skipping duplicate file $basename from command line"
- }
- continue
- }
- if {$basename != "dbCommon.dbd"} { lappend filesDone $basename }
- set file [open $filename]
- set FileName($file) $filename
- scanfile $global_context $file
- close $file
-}
diff --git a/tests/test_expand_dbd.py b/tests/test_expand_dbd.py
index eae44dee..f1384d6d 100644
--- a/tests/test_expand_dbd.py
+++ b/tests/test_expand_dbd.py
@@ -8,33 +8,33 @@ import pytest
@pytest.fixture
def expanddbdtcl():
require_path = Path(os.environ.get("E3_REQUIRE_LOCATION"))
- expanddbdtcl = require_path / "tools" / "expandDBD.tcl"
+ expanddbdtcl = require_path / "tools" / "expandDBD"
assert expanddbdtcl.is_file() and os.access(expanddbdtcl, os.X_OK)
return expanddbdtcl
-def test_missing_dbd_file(tmpdir, expanddbdtcl):
- dbd_file = Path(tmpdir) / "tmp.dbd"
+def test_missing_dbd_file(tmp_path, expanddbdtcl):
+ dbd_file = tmp_path / "tmp.dbd"
dbd_file.write_text("include not_a_file.dbd")
result = subprocess.run(
- [expanddbdtcl, dbd_file],
+ [expanddbdtcl, "-I", tmp_path, dbd_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
)
assert result.returncode == 1
- assert "file not_a_file.dbd not found" in result.stderr
+ assert "File 'not_a_file.dbd' not found" in result.stderr
-def test_include_dbd_file(tmpdir, expanddbdtcl):
- dbd_a = Path(tmpdir) / "a.dbd"
+def test_include_dbd_file(tmp_path, expanddbdtcl):
+ dbd_a = tmp_path / "a.dbd"
dbd_a.write_text("include b.dbd")
- dbd_b = Path(tmpdir) / "b.dbd"
+ dbd_b = tmp_path / "b.dbd"
dbd_b.write_text("content")
result = subprocess.run(
- [expanddbdtcl, "-I", str(tmpdir), dbd_a],
+ [expanddbdtcl, "-I", str(tmp_path), dbd_a],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
@@ -43,18 +43,48 @@ def test_include_dbd_file(tmpdir, expanddbdtcl):
assert "content" == result.stdout.strip("\n")
-def test_skip_repeated_includes(tmpdir, expanddbdtcl):
- dbd_a = Path(tmpdir) / "a.dbd"
+def test_skip_repeated_includes(tmp_path, expanddbdtcl):
+ dbd_a = tmp_path / "a.dbd"
dbd_a.write_text("include b.dbd\ninclude b.dbd")
- dbd_b = Path(tmpdir) / "b.dbd"
+ dbd_b = tmp_path / "b.dbd"
dbd_b.touch()
result = subprocess.run(
- [expanddbdtcl, "-I", str(tmpdir), dbd_a],
+ [expanddbdtcl, "-I", str(tmp_path), dbd_a],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
)
assert result.returncode == 0
assert "Info: skipping duplicate file b.dbd included from" in result.stderr
+
+
+def test_record_names_from_dbds(tmp_path, expanddbdtcl):
+ dbd_a = tmp_path / "a.dbd"
+ dbd_a.write_text("include aRecord.dbd")
+
+ dbd_record = tmp_path / "aRecord.dbd"
+ dbd_record.touch()
+
+ result = subprocess.run(
+ [expanddbdtcl, "-r", "-I", str(tmp_path), dbd_a],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding="utf-8",
+ )
+ assert result.returncode == 0
+ assert result.stdout.splitlines() == ["a"]
+
+
+def test_missing_record_names_from_dbds(tmp_path, expanddbdtcl):
+ dbd_a = tmp_path / "a.dbd"
+ dbd_a.write_text("include aRecord.dbd")
+
+ result = subprocess.run(
+ [expanddbdtcl, "-r", "-I", str(tmp_path), dbd_a],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding="utf-8",
+ )
+ assert result.returncode == 1
--
GitLab