Replace expandDBD.tcl with python script

The purpose of this is to remove the dependency on tclx, a non-standard
package.

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### New Features
+
+### Bugfixes
+
+### Other changes
+
+* Replaced `tclx` script to expand .dbd files with a python script
+
 ## [5.0.0]
 
 ### New Features

configure/module/RULES_REQUIRE

@@ -23,7 +23,7 @@ endif
 install: requireconf
 
 requireconf: e3-site-path e3-require-path
-	$(QUIET) install    -m 755 $(wildcard $(E3_MODULE_SRC_PATH)/tools/*.tcl)  $(E3_REQUIRE_TOOLS)/
+	$(QUIET) install    -m 755 $(wildcard $(E3_MODULE_SRC_PATH)/tools/expandDBD)  $(E3_REQUIRE_TOOLS)/
 	$(QUIET) install    -m 644 $(E3_MODULE_SRC_PATH)/tools/driver.makefile    $(E3_REQUIRE_TOOLS)/
 	$(QUIET) install    -m 755 $(E3_MODULE_SRC_PATH)/tools/revision_number       $(E3_REQUIRE_TOOLS)/
 	$(QUIET) install    -m 755 $(E3_SHELL_FILES)             $(E3_REQUIRE_BIN)/

require-ess/tools/driver.makefile

@@ -216,7 +216,7 @@ export DBD_SRCS
 #record dbd files given in DBDS
 RECORDS1 = $(patsubst %Record.dbd, %, $(filter-out dev%, $(filter %Record.dbd, $(notdir ${DBD_SRCS}))))
 #record dbd files included by files given in DBDS
-RECORDS2 = $(filter-out dev%, $(shell ${MAKEHOME}/expandDBD.tcl -r $(addprefix -I, $(sort $(dir ${DBD_SRCS}))) $(realpath ${DBDS})))
+RECORDS2 = $(filter-out dev%, $(shell ${MAKEHOME}/expandDBD -r $(addprefix -I, $(sort $(dir ${DBD_SRCS}))) $(realpath ${DBDS}) 2> /dev/null))
 RECORDS = $(sort ${RECORDS1} ${RECORDS2})
 export RECORDS
 
@@ -610,7 +610,7 @@ MODULEINFOS:
 # because it has too strict checks to be used for a loadable module.
 ${MODULEDBD}: ${DBDFILES}
 	@echo "Expanding $@"
-	${MAKEHOME}expandDBD.tcl -$(basename ${EPICSVERSION}) ${DBDEXPANDPATH} $^ > $@
+	${MAKEHOME}expandDBD ${DBDEXPANDPATH} $^ > $@
 
 # Install everything.
 INSTALL_LIBS = ${MODULELIB:%=${INSTALL_LIB}/%}

require-ess/tools/expandDBD

+#!/usr/bin/env python
+from __future__ import print_function
+
+import argparse
+import os
+import re
+import sys
+
+SCANNED_FILES = set()
+RECORD_REGEX = re.compile(r"include[ \t]*\"?((.*)Record.dbd)\"?")
+INCLUDE_REGEX = re.compile(r"include[ \t]+\"?([^\"]*)\"?")
+BLANK_LINE_REGEX = re.compile(r"^[ \t]*(#|%|$)")
+
+
+def find_dbd_file(filename, include_paths):
+    """Search for a given .dbd file among the include paths."""
+    for include_path in include_paths:
+        potential_path = os.path.join(include_path, filename)
+        if os.path.exists(potential_path):
+            return potential_path
+    return None
+
+
+def open_dbd_file(current_file, filename, includes=None):
+    """Open a .dbd file and return the lines."""
+    if not includes:
+        includes = ["."]
+
+    basename = os.path.basename(filename)
+    if basename in SCANNED_FILES:
+        print(
+            "Info: skipping duplicate file {basename} included from {current_file}".format(
+                basename=basename, current_file=current_file
+            ),
+            file=sys.stderr,
+        )
+        return []
+    dbd_file = find_dbd_file(basename, includes)
+    if dbd_file is None:
+        print("File '{basename}' not found".format(basename=basename), file=sys.stderr)
+        sys.exit(1)
+    SCANNED_FILES.add(basename)
+    with open(dbd_file, "r") as f:
+        return [line.strip() for line in f.readlines()]
+
+
+def expand_dbd_file(current_file, dbdlines, includes):
+    """Print to stdout a fully-expanded .dbd file."""
+    for line in dbdlines:
+        if BLANK_LINE_REGEX.match(line):
+            continue
+        m = INCLUDE_REGEX.search(line)
+        if m:
+            lines = open_dbd_file(current_file, m.group(1), includes)
+            expand_dbd_file(m.group(1), lines, includes)
+        else:
+            print(line)
+
+
+def parse_record_dbd_file(current_file, dbdlines, includes):
+    """Recursively search through .dbd files for xRecord.dbd, and print the record names."""
+    for line in dbdlines:
+        if BLANK_LINE_REGEX.match(line):
+            continue
+        m = RECORD_REGEX.search(line)
+        if m:
+            if find_dbd_file(m.group(1), includes):
+                print(m.group(2))
+            else:
+                print(
+                    "File '{group}' not found".format(group=m.group(1)), file=sys.stderr
+                )
+                sys.exit(1)
+            continue
+        m = INCLUDE_REGEX.search(line)
+        if m:
+            lines = open_dbd_file(current_file, m.group(1), includes)
+            parse_record_dbd_file(m.group(1), lines, includes)
+
+
+def handle_dbd_file(dbdfiles, include, record_types):
+    """Process the .dbd files."""
+    for dbdfile in dbdfiles:
+        lines = open_dbd_file("command line", dbdfile, include)
+        if record_types:
+            parse_record_dbd_file(dbdfile, lines, include)
+        else:
+            expand_dbd_file(dbdfile, lines, include)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-I", "--include", action="append")
+    parser.add_argument("-r", "--record-types", action="store_true")
+    parser.add_argument("dbdfiles", nargs="+")
+
+    args = parser.parse_args()
+
+    handle_dbd_file(**vars(args))
+
+
+if __name__ == "__main__":
+    main()

require-ess/tools/expandDBD.tcl

-#!/usr/bin/env tclsh
-
-package require Tclx
-
-set global_context [scancontext create]
-
-set epicsversion 3.14
-set quiet 0
-set recordtypes 0
-set seachpath {}
-set filesDone {}
-
-while {[llength $argv]} {
-    switch -glob -- [lindex $argv 0] {
-        "-[0-9]*" { set epicsversion [string range [lindex $argv 0] 1 end]}
-        "-q"      { set quiet 1 }
-        "-r"      { set recordtypes 1; set quiet 1 }
-        "-I"      { lappend seachpath [lindex $argv 1]; set argv [lreplace $argv 0 1]; continue }
-        "-I*"     { lappend seachpath [string range [lindex $argv 0] 2 end] }
-        "--"      { set argv [lreplace $argv 0 0]; break }
-        "-*"      { puts stderr "Warning: Unknown option [lindex $argv 0] ignored" }
-        default   { break }
-    }
-    set argv [lreplace $argv 0 0]
-}
-
-proc opendbd {name} {
-    global seachpath
-    foreach dir $seachpath {
-        if ![catch {
-            set file [open [file join $dir $name]]
-        }] {
-            return $file
-        }
-    }
-    return -code error "file $name not found"
-}
-
-scanmatch $global_context {^[ \t]*(#|%|$)} {
-    continue
-}
-
-if {$recordtypes} {
-    scanmatch $global_context {include[ \t]+"?((.*)Record.dbd)"?} {
-    if ![catch {
-        close [opendbd $matchInfo(submatch0)]
-    }] {
-        puts $matchInfo(submatch1)
-    }
-    continue
-}
-
-} else {
-
-    scanmatch $global_context {(registrar|variable|function)[ \t]*\([ \t]*"?([a-zA-Z0-9_]+)"?[ \t]*\)} {
-        global epicsversion
-        if {$epicsversion > 3.13} {puts $matchInfo(submatch0)($matchInfo(submatch1))}
-    }
-    scanmatch $global_context {variable[ \t]*\([ \t]*"?([a-zA-Z0-9_]+)"?[ \t]*,[ \t]*"?([a-zA-Z0-9_]+)"?[ \t]*\)} {
-        global epicsversion
-        if {$epicsversion > 3.13} {puts variable($matchInfo(submatch0),$matchInfo(submatch1))}
-    }
-
-    scanmatch $global_context {
-        puts $matchInfo(line)
-    }
-}
-
-scanmatch $global_context {include[ \t]+"?([^"]*)"?} {
-    global seachpath
-    global FileName
-    global quiet
-    if [catch {
-        includeFile $global_context $matchInfo(submatch0)
-    } msg] {
-        if {!$quiet} {
-            puts stderr "ERROR: $msg in path \"$seachpath\" called from $FileName($matchInfo(handle)) line $matchInfo(linenum)"
-            exit 1
-        }
-    }
-    continue
-}
-
-proc includeFile {context filename} {
-    global global_context FileName filesDone matchInfo quiet
-    set basename [file tail $filename]
-    if {[lsearch $filesDone $basename ] != -1} {
-        if {!$quiet} {
-            puts stderr "Info: skipping duplicate file $basename included from $FileName($matchInfo(handle))"
-        }
-        return
-    }
-    if {$filename != "dbCommon.dbd"} { lappend filesDone [file tail $filename] }
-    set file [opendbd $filename]
-    set FileName($file) $filename
-    #puts "#include $filename from $FileName($matchInfo(handle))"
-    scanfile $context $file
-    close $file
-}
-
-foreach filename $argv {
-    global filesDone quiet
-    set basename [file tail $filename]
-    if {[lsearch $filesDone $basename] != -1} {
-        if {!$quiet} {
-            puts stderr "Info: skipping duplicate file $basename from command line"
-        }
-        continue
-    }
-    if {$basename != "dbCommon.dbd"} { lappend filesDone $basename }
-    set file [open $filename]
-    set FileName($file) $filename
-    scanfile $global_context $file
-    close $file
-}

tests/test_expand_dbd.py

@@ -8,33 +8,33 @@ import pytest
 @pytest.fixture
 def expanddbdtcl():
     require_path = Path(os.environ.get("E3_REQUIRE_LOCATION"))
-    expanddbdtcl = require_path / "tools" / "expandDBD.tcl"
+    expanddbdtcl = require_path / "tools" / "expandDBD"
     assert expanddbdtcl.is_file() and os.access(expanddbdtcl, os.X_OK)
     return expanddbdtcl
 
 
-def test_missing_dbd_file(tmpdir, expanddbdtcl):
-    dbd_file = Path(tmpdir) / "tmp.dbd"
+def test_missing_dbd_file(tmp_path, expanddbdtcl):
+    dbd_file = tmp_path / "tmp.dbd"
     dbd_file.write_text("include not_a_file.dbd")
     result = subprocess.run(
-        [expanddbdtcl, dbd_file],
+        [expanddbdtcl, "-I", tmp_path, dbd_file],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         encoding="utf-8",
     )
     assert result.returncode == 1
-    assert "file not_a_file.dbd not found" in result.stderr
+    assert "File 'not_a_file.dbd' not found" in result.stderr
 
 
-def test_include_dbd_file(tmpdir, expanddbdtcl):
-    dbd_a = Path(tmpdir) / "a.dbd"
+def test_include_dbd_file(tmp_path, expanddbdtcl):
+    dbd_a = tmp_path / "a.dbd"
     dbd_a.write_text("include b.dbd")
 
-    dbd_b = Path(tmpdir) / "b.dbd"
+    dbd_b = tmp_path / "b.dbd"
     dbd_b.write_text("content")
 
     result = subprocess.run(
-        [expanddbdtcl, "-I", str(tmpdir), dbd_a],
+        [expanddbdtcl, "-I", str(tmp_path), dbd_a],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         encoding="utf-8",
@@ -43,18 +43,48 @@ def test_include_dbd_file(tmpdir, expanddbdtcl):
     assert "content" == result.stdout.strip("\n")
 
 
-def test_skip_repeated_includes(tmpdir, expanddbdtcl):
-    dbd_a = Path(tmpdir) / "a.dbd"
+def test_skip_repeated_includes(tmp_path, expanddbdtcl):
+    dbd_a = tmp_path / "a.dbd"
     dbd_a.write_text("include b.dbd\ninclude b.dbd")
 
-    dbd_b = Path(tmpdir) / "b.dbd"
+    dbd_b = tmp_path / "b.dbd"
     dbd_b.touch()
 
     result = subprocess.run(
-        [expanddbdtcl, "-I", str(tmpdir), dbd_a],
+        [expanddbdtcl, "-I", str(tmp_path), dbd_a],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         encoding="utf-8",
     )
     assert result.returncode == 0
     assert "Info: skipping duplicate file b.dbd included from" in result.stderr
+
+
+def test_record_names_from_dbds(tmp_path, expanddbdtcl):
+    dbd_a = tmp_path / "a.dbd"
+    dbd_a.write_text("include aRecord.dbd")
+
+    dbd_record = tmp_path / "aRecord.dbd"
+    dbd_record.touch()
+
+    result = subprocess.run(
+        [expanddbdtcl, "-r", "-I", str(tmp_path), dbd_a],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="utf-8",
+    )
+    assert result.returncode == 0
+    assert result.stdout.splitlines() == ["a"]
+
+
+def test_missing_record_names_from_dbds(tmp_path, expanddbdtcl):
+    dbd_a = tmp_path / "a.dbd"
+    dbd_a.write_text("include aRecord.dbd")
+
+    result = subprocess.run(
+        [expanddbdtcl, "-r", "-I", str(tmp_path), dbd_a],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="utf-8",
+    )
+    assert result.returncode == 1