custom_usermod improvements

pio-scripts/dynarray.py

@@ -0,0 +1,19 @@
+# Add a section to the linker script to store our dynamic arrays
+# This is implemented as a pio post-script to ensure that we can
+# place our linker script at the correct point in the command arguments.
+Import("env")
+from pathlib import Path
+
+platform = env.get("PIOPLATFORM")
+script_file = Path(f"tools/dynarray_{platform}.ld")
+if script_file.is_file():
+    linker_script = f"-T{script_file}"
+    if platform == "espressif32":
+        # For ESP32, the script must be added at the right point in the list
+        linkflags = env.get("LINKFLAGS", [])
+        idx = linkflags.index("memory.ld")    
+        linkflags.insert(idx+1, linker_script)    
+        env.Replace(LINKFLAGS=linkflags)
+    else:
+        # For other platforms, put it in last
+        env.Append(LINKFLAGS=[linker_script])

pio-scripts/load_usermods.py

@@ -1,6 +1,8 @@
 Import('env')
 from collections import deque
 from pathlib import Path   # For OS-agnostic path manipulation
+import re
+from urllib.parse import urlparse
 from click import secho
 from SCons.Script import Exit
 from platformio.builder.tools.piolib import LibBuilderBase
@@ -25,25 +27,117 @@ def find_usermod(mod: str) -> Path:
     return mp
   raise RuntimeError(f"Couldn't locate module {mod} in usermods directory!")
 
-def is_wled_module(dep: LibBuilderBase) -> bool:
-  """Returns true if the specified library is a wled module
+# Names of external/registry deps listed in custom_usermods.
+# Populated during parsing below; read by is_wled_module() at configure time.
+_custom_usermod_names: set[str] = set()
+
+# Matches any RFC-valid URL scheme (http, https, git, git+https, symlink, file, hg+ssh, etc.)
+_URL_SCHEME_RE = re.compile(r'^[a-zA-Z][a-zA-Z0-9+.-]*://')
+# SSH git URL: user@host:path  (e.g. git@github.com:user/repo.git#tag)
+_SSH_URL_RE = re.compile(r'^[^@\s]+@[^@:\s]+:[^:\s]')
+# Explicit custom name: "LibName = <spec>"  (PlatformIO [<name>=]<spec> form)
+_NAME_EQ_RE = re.compile(r'^([A-Za-z0-9_.-]+)\s*=\s*(\S.*)')
+
+
+def _is_external_entry(line: str) -> bool:
+  """Return True if line is a lib_deps-style external/registry entry."""
+  if _NAME_EQ_RE.match(line):              # "LibName = <spec>"
+    return True
+  if _URL_SCHEME_RE.match(line):           # https://, git://, symlink://, etc.
+    return True
+  if _SSH_URL_RE.match(line):              # git@github.com:user/repo.git
+    return True
+  if '@' in line:                          # "owner/Name @ ^1.0.0"
+    return True
+  if re.match(r'^[^/\s]+/[^/\s]+$', line):  # "owner/Name"
+    return True
+  return False
+
+
+def _predict_dep_name(entry: str) -> str | None:
+  """Predict the library name PlatformIO will assign to this dep (best-effort).
+
+  Accuracy relies on the library's manifest "name" matching the repo/package
+  name in the spec. This holds for well-authored libraries; the libArchive
+  check (which requires library.json) provides an early-failure safety net.
   """
-  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
-
-## Script starts here
-# Process usermod option
-usermods = env.GetProjectOption("custom_usermods","")
+  entry = entry.strip()
+  # "LibName = <spec>" — name is given explicitly; always use it
+  m = _NAME_EQ_RE.match(entry)
+  if m:
+    return m.group(1).strip()
+  # URL scheme: extract name from path
+  if _URL_SCHEME_RE.match(entry):
+    parsed = urlparse(entry)
+    if parsed.netloc in ('github.com', 'gitlab.com', 'bitbucket.com'):
+      parts = [p for p in parsed.path.split('/') if p]
+      if len(parts) >= 2:
+        name = parts[1]
+      else:
+        name = Path(parsed.path.rstrip('/')).name.strip()
+      if name.endswith('.git'):
+        name = name[:-4]
+      return name or None
+  # SSH git URL: git@github.com:user/repo.git#tag → repo
+  if _SSH_URL_RE.match(entry):
+    path_part = entry.split(':', 1)[1].split('#')[0].rstrip('/')
+    name = Path(path_part).name
+    return (name[:-4] if name.endswith('.git') else name) or None
+  # Versioned registry: "owner/Name @ version" → Name
+  if '@' in entry:
+    name_part = entry.split('@')[0].strip()
+    return name_part.split('/')[-1].strip() if '/' in name_part else name_part
+  # Plain registry: "owner/Name" → Name
+  if re.match(r'^[^/\s]+/[^/\s]+$', entry):
+    return entry.split('/')[-1].strip()
+  return None
 
-# Handle "all usermods" case
-if usermods == '*':
-  usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
-else:
-  usermods = usermods.split()
 
-if usermods:
-  # Inject usermods in to project lib_deps
-  symlinks = [f"symlink://{find_usermod(mod).resolve()}" for mod in usermods]
-  env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + symlinks)
+def is_wled_module(dep: LibBuilderBase) -> bool:
+  """Returns true if the specified library is a wled module."""
+  return (
+    usermod_dir in Path(dep.src_dir).parents
+    or str(dep.name).startswith("wled-")
+    or dep.name in _custom_usermod_names
+  )
+
+
+## Script starts here — parse custom_usermods
+raw_usermods = env.GetProjectOption("custom_usermods", "")
+usermods_libdeps: list[str] = []
+
+for line in raw_usermods.splitlines():
+  line = line.strip()
+  if not line or line.startswith('#') or line.startswith(';'):
+    continue
+
+  if _is_external_entry(line):
+    # External URL or registry entry: pass through to lib_deps unchanged.
+    predicted = _predict_dep_name(line)
+    if predicted:
+      _custom_usermod_names.add(predicted)
+    else:
+      secho(
+        f"WARNING: Cannot determine library name for custom_usermods entry "
+        f"{line!r}. If it is not recognised as a WLED module at build time, "
+        f"ensure its library.json 'name' matches the repo name.",
+        fg="yellow", err=True)
+    usermods_libdeps.append(line)
+  else:
+    # Bare name(s): split on whitespace for backwards compatibility.
+    for token in line.split():
+      if token == '*':
+        for mod_path in sorted(usermod_dir.iterdir()):
+          if mod_path.is_dir() and (mod_path / 'library.json').exists():
+            _custom_usermod_names.add(mod_path.name)
+            usermods_libdeps.append(f"symlink://{mod_path.resolve()}")
+      else:
+        resolved = find_usermod(token)
+        _custom_usermod_names.add(resolved.name)
+        usermods_libdeps.append(f"symlink://{resolved.resolve()}")
+
+if usermods_libdeps:
+  env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + usermods_libdeps)
 
 # Utility function for assembling usermod include paths
 def cached_add_includes(dep, dep_cache: set, includes: deque):
@@ -86,16 +180,25 @@ def wrapped_ConfigureProjectLibBuilder(xenv):
     # Add WLED's own dependencies
     for dir in extra_include_dirs:
       dep.env.PrependUnique(CPPPATH=str(dir))
+    # Ensure debug info is emitted for this module's source files.
+    # validate_modules.py uses `nm --defined-only -l` on the final ELF to check
+    # that each module has at least one symbol placed in the binary.  The -l flag
+    # reads DWARF debug sections to map placed symbols back to their original source
+    # files; without -g those sections are absent and the check cannot attribute any
+    # symbol to a specific module.  We scope this to usermods only — the main WLED
+    # build and other libraries are unaffected.
+    dep.env.AppendUnique(CCFLAGS=["-g"])
     # Enforce that libArchive is not set; we must link them directly to the executable
     if dep.lib_archive:
       broken_usermods.append(dep)
 
   if broken_usermods:
     broken_usermods = [usermod.name for usermod in broken_usermods]
     secho(
-      f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly",
-      fg="red",
-      err=True)
+      f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- "
+      f"modules will not compile in correctly. Add '\"build\": {{\"libArchive\": false}}' "
+      f"to their library.json.",
+      fg="red", err=True)
     Exit(1)
 
   # Save the depbuilders list for later validation

pio-scripts/validate_modules.py

@@ -1,16 +1,10 @@
+import os
 import re
+import subprocess
 from pathlib import Path   # For OS-agnostic path manipulation
-from typing import Iterable
 from click import secho
 from SCons.Script import Action, Exit
-from platformio.builder.tools.piolib import LibBuilderBase
-
-
-def is_wled_module(env, dep: LibBuilderBase) -> bool:
-  """Returns true if the specified library is a wled module
-  """
-  usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
-  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
+Import("env")
 
 
 def read_lines(p: Path):
@@ -19,29 +13,71 @@ def read_lines(p: Path):
         return f.readlines()
 
 
-def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]:
-    """ Identify which dirs contributed to the final build
+def _get_nm_path(env) -> str:
+    """ Derive the nm tool path from the build environment """
+    if "NM" in env:
+        return env.subst("$NM")
+    # Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-nm
+    cc = env.subst("$CC")
+    nm = re.sub(r'(gcc|g\+\+)$', 'nm', os.path.basename(cc))
+    return os.path.join(os.path.dirname(cc), nm)
+
+
+def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]:
+    """ Check which modules have at least one defined symbol placed in the ELF.
 
-        Returns the (sub)set of dirs that are found in the output ELF
+        The map file is not a reliable source for this: with LTO, original object
+        file paths are replaced by temporary ltrans.o partitions in all output
+        sections, making per-module attribution impossible from the map alone.
+        Instead we invoke nm --defined-only -l on the ELF, which uses DWARF debug
+        info to attribute each placed symbol to its original source file.
+
+        Requires usermod libraries to be compiled with -g so that DWARF sections
+        are present in the ELF.  load_usermods.py injects -g for all WLED modules
+        via dep.env.AppendUnique(CCFLAGS=["-g"]).
+
+        Returns the set of build_dir basenames for confirmed modules.
     """
-    # Pattern to match symbols in object directories
-    # Join directories into alternation
-    usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs])
-    # Matches nonzero address, any size, and any path in a matching directory
-    object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
+    nm_path = _get_nm_path(env)
+    try:
+        result = subprocess.run(
+            [nm_path, "--defined-only", "-l", str(elf_path)],
+            capture_output=True, text=True, errors="ignore", timeout=120,
+        )
+        nm_output = result.stdout
+    except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e:
+        secho(f"WARNING: nm failed ({e}); skipping per-module validation", fg="yellow", err=True)
+        return {Path(b.build_dir).name for b in module_lib_builders}  # conservative pass
+
+    # Build a filtered set of lines that have a nonzero address.
+    # nm --defined-only still includes debugging symbols (type 'N') such as the
+    # per-CU markers GCC emits in .debug_info (e.g. "usermod_example_cpp_6734d48d").
+    # These live at address 0x00000000 in their debug section — not in any load
+    # segment — so filtering them out leaves only genuinely placed symbols.
+    placed_lines = [
+        line for line in nm_output.splitlines()
+        if (parts := line.split(None, 1)) and parts[0].lstrip('0')
+    ]
+    placed_output = "\n".join(placed_lines)
 
     found = set()
-    for line in map_file:
-        matches = object_path_regex.findall(line)
-        for m in matches:
-            found.add(m)
+    for builder in module_lib_builders:
+        # builder.src_dir is the library source directory (used by is_wled_module() too)
+        src_dir = str(builder.src_dir).rstrip("/\\")
+        # Guard against prefix collisions (e.g. /path/to/mod vs /path/to/mod-extra)
+        # by requiring a path separator immediately after the directory name.
+        if re.search(re.escape(src_dir) + r'[/\\]', placed_output):
+            found.add(Path(builder.build_dir).name)
     return found
 
 
+DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray"
+USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1"
+
 def count_usermod_objects(map_file: list[str]) -> int:
     """ Returns the number of usermod objects in the usermod list """
     # Count the number of entries in the usermods table section
-    return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
+    return len([x for x in map_file if USERMODS_SECTION in x])
 
 
 def validate_map_file(source, target, env):
@@ -65,16 +101,17 @@ def validate_map_file(source, target, env):
     usermod_object_count = count_usermod_objects(map_file_contents)
     secho(f"INFO: {usermod_object_count} usermod object entries")
 
-    confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())
+    elf_path = build_dir / env.subst("${PROGNAME}.elf")
+    confirmed_modules = check_elf_modules(elf_path, env, module_lib_builders)
+
     missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
     if missing_modules:
         secho(
-            f"ERROR: No object files from {missing_modules} found in linked output!",
+            f"ERROR: No symbols from {missing_modules} found in linked output!",
             fg="red",
             err=True)
         Exit(1)
     return None
 
-Import("env")
 env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
 env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

platformio.ini

@@ -135,6 +135,7 @@ extra_scripts =
   pre:pio-scripts/set_metadata.py
   post:pio-scripts/output_bins.py
   post:pio-scripts/strip-floats.py
+  post:pio-scripts/dynarray.py
   pre:pio-scripts/user_config_copy.py
   pre:pio-scripts/load_usermods.py
   pre:pio-scripts/build_ui.py

platformio_override.sample.ini

@@ -538,9 +538,21 @@ monitor_filters = esp32_exception_decoder
 # 433MHz RF remote example for esp32dev
 [env:esp32dev_usermod_RF433]
 extends = env:esp32dev
-build_flags = ${env:esp32dev.build_flags} -D USERMOD_RF433
-lib_deps = ${env:esp32dev.lib_deps}
-  sui77/rc-switch @ 2.6.4
+custom_usermods =
+  ${env:esp32dev.custom_usermods}
+  RF433
+
+# External usermod from a git repository.
+# The library's `library.json` must include `"build": {"libArchive": false}`.
+# The name PlatformIO assigns is taken from the library's `library.json` "name" field.
+# If that name doesn't match the repo name in the URL, use the "LibName = URL" form
+# shown in the commented-out line below to supply the name explicitly.
+[env:esp32dev_external_usermod]
+extends = env:esp32dev
+custom_usermods =
+  ${env:esp32dev.custom_usermods}
+  https://github.com/wled/wled-usermod-example.git#main
+
 
 # ------------------------------------------------------------------------------
 # Hub75 examples

tools/dynarray_espressif32.ld

@@ -0,0 +1,10 @@
+/* ESP32 linker script fragment to add dynamic array section to binary */
+SECTIONS
+{
+  .dynarray :
+  {
+    . = ALIGN(0x10);
+    KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*)))
+  } > default_rodata_seg
+}
+INSERT AFTER .flash.rodata;

wled00/dynarray.h

@@ -0,0 +1,34 @@
+/* dynarray.h
+
+Macros for generating a "dynamic array", a static array of objects declared in different translation units
+
+*/
+
+#pragma once
+
+// Declare the beginning and ending elements of a dynamic array of 'type'.
+// This must be used in only one translation unit in your program for any given array.
+#define DECLARE_DYNARRAY(type, array_name) \
+  static type const DYNARRAY_BEGIN(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".0"), unused)) = {}; \
+  static type const DYNARRAY_END(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".99999"), unused)) = {};
+
+// Declare an object that is a member of a dynamic array.  "member name" must be unique; "array_section" is an integer for ordering items.
+// It is legal to define multiple items with the same section name; the order of those items will be up to the linker.
+#define DYNARRAY_MEMBER(type, array_name, member_name, array_section) type const member_name __attribute__((__section__(DYNARRAY_SECTION "." #array_name "." #array_section), used))
+
+#define DYNARRAY_BEGIN(array_name) array_name##_begin
+#define DYNARRAY_END(array_name) array_name##_end
+#define DYNARRAY_LENGTH(array_name) (&DYNARRAY_END(array_name)[0] - &DYNARRAY_BEGIN(array_name)[0])
+
+#ifdef ESP8266
+// ESP8266 linker script cannot be extended with a unique section for dynamic arrays.
+// We instead pack them in the ".dtors" section, as it's sorted and uploaded to the flash
+// (but will never be used in the embedded system)
+#define DYNARRAY_SECTION ".dtors"
+
+#else /* ESP8266 */
+
+// Use a unique named section; the linker script must be extended to ensure it's correctly placed.
+#define DYNARRAY_SECTION ".dynarray"
+
+#endif