Add Static Module Support

CMakeLists.txt

@@ -14,6 +14,7 @@ endif ()
 
 # Options
 option(BUILD_LUA "Build Valkey Lua scripting engine" ON)
+option(BUILD_STATIC_LUA "Build Valkey Lua scripting engine as a static module" OFF)
 option(BUILD_UNIT_GTESTS "Build valkey-unit-gtests" OFF)
 option(BUILD_TEST_MODULES "Build all test modules" OFF)
 option(BUILD_EXAMPLE_MODULES "Build example modules" OFF)
@@ -35,6 +36,7 @@ include(Packaging)
 
 # Clear cached variables from the cache
 unset(BUILD_LUA CACHE)
+unset(BUILD_STATIC_LUA CACHE)
 unset(BUILD_TESTS CACHE)
 unset(CLANGPP CACHE)
 unset(CLANG CACHE)

src/CMakeLists.txt

@@ -19,22 +19,32 @@ add_dependencies(valkey-server release_header)
 
 if (BUILD_LUA)
     message(STATUS "Build Lua scripting engine module")
+    if (BUILD_STATIC_LUA)
+        add_compile_definitions(STATIC_LUA=1)
+        message(STATUS "Building LUA as a STATIC module")
+    else ()
+        add_compile_definitions(STATIC_LUA=0)
+        message(STATUS "Building LUA as a DYNAMIC module")
+    endif ()
     add_subdirectory(modules/lua)
-    add_dependencies(valkey-server valkeylua)
+    if (BUILD_STATIC_LUA)
+        target_link_libraries(valkey-server $<LINK_LIBRARY:WHOLE_ARCHIVE,valkeylua>)
+    else ()
+        add_dependencies(valkey-server valkeylua)
+    endif ()
     target_compile_definitions(valkey-server PRIVATE LUA_ENABLED)
     if (UNIX AND NOT APPLE)
         target_compile_definitions(valkey-server PRIVATE LUA_LIB=libvalkeylua.so)
-        target_link_options(valkey-server PRIVATE -Wl,--disable-new-dtags)
     else ()
         target_compile_definitions(valkey-server PRIVATE LUA_LIB=libvalkeylua.dylib)
     endif ()
     set(VALKEY_INSTALL_RPATH "")
-    set_target_properties(valkey-server PROPERTIES
-        INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_LIBRARY_OUTPUT_DIRECTORY}"
-        INSTALL_RPATH_USE_LINK_PATH TRUE
-        BUILD_WITH_INSTALL_RPATH TRUE
-    )
-endif()
+    set_target_properties(
+        valkey-server
+        PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_LIBRARY_OUTPUT_DIRECTORY}"
+                   INSTALL_RPATH_USE_LINK_PATH TRUE
+                   BUILD_WITH_INSTALL_RPATH TRUE)
+endif ()
 unset(BUILD_LUA CACHE)
 
 if (VALKEY_RELEASE_BUILD)
@@ -49,9 +59,8 @@ if (DEBUG_FORCE_DEFRAG)
 endif ()
 
 if (BUILD_SANITIZER)
-    # 'BUILD_SANITIZER' is defined in ValkeySetup module (based on user input)
-    # If defined, the variables 'VALKEY_SANITAIZER_CFLAGS' and 'VALKEY_SANITAIZER_LDFLAGS'
-    # are set with the link & compile flags required
+    # 'BUILD_SANITIZER' is defined in ValkeySetup module (based on user input) If defined, the variables
+    # 'VALKEY_SANITAIZER_CFLAGS' and 'VALKEY_SANITAIZER_LDFLAGS' are set with the link & compile flags required
     message(STATUS "Adding sanitizer flags for target valkey-server")
     target_compile_options(valkey-server PRIVATE ${VALKEY_SANITAIZER_CFLAGS})
     target_link_options(valkey-server PRIVATE ${VALKEY_SANITAIZER_LDFLAGS})
@@ -118,10 +127,10 @@ endif ()
 
 # Friendly hint like the Makefile one
 file(RELATIVE_PATH _CMAKE_DIR_RELATIVE_PATH "${CMAKE_SOURCE_DIR}" "${CMAKE_BINARY_DIR}")
-add_custom_target(hint ALL
+add_custom_target(
+    hint ALL
     DEPENDS valkey-server valkey-cli valkey-benchmark
     COMMAND ${CMAKE_COMMAND} -E echo ""
     COMMAND ${CMAKE_COMMAND} -E echo "Hint: It is a good idea to run tests with your CMake-built binaries \\;\\)"
     COMMAND ${CMAKE_COMMAND} -E echo "      ./${_CMAKE_DIR_RELATIVE_PATH}/runtest"
-    COMMAND ${CMAKE_COMMAND} -E echo ""
-)
+    COMMAND ${CMAKE_COMMAND} -E echo "")

src/Makefile

@@ -272,6 +272,19 @@ else
 endif
 endif
 
+# Build Lua module as static library instead of shared
+ifeq ($(STATIC_LUA),yes)
+	FINAL_CFLAGS+=-DSTATIC_LUA=1
+	ifeq ($(uname_S),Darwin)
+		LUA_LDFLAGS=-Wl,-export_dynamic -Wl,-force_load,$(current_dir)/modules/lua/libvalkeylua.a ../deps/lua/src/liblua.a
+	else
+		LUA_LDFLAGS=-Wl,--whole-archive $(current_dir)/modules/lua/libvalkeylua.a -Wl,--no-whole-archive ../deps/lua/src/liblua.a
+	endif
+else
+	FINAL_CFLAGS+=-DSTATIC_LUA=0
+	LUA_LDFLAGS=
+endif
+
 # Determine systemd support and/or build preference (defaulting to auto-detection)
 BUILD_WITH_SYSTEMD=no
 LIBSYSTEMD_LIBS=-lsystemd
@@ -688,7 +701,7 @@ endif
 
 # valkey-server
 $(SERVER_NAME): $(ENGINE_SERVER_OBJ) $(LUA_MODULE)
-	$(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
+	$(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(LUA_LDFLAGS)
 
 # Valkey static library, used to compile against for unit testing
 $(ENGINE_LIB_NAME): $(ENGINE_SERVER_OBJ)
@@ -716,7 +729,7 @@ $(RDMA_MODULE_NAME): $(SERVER_NAME)
 
 # engine_lua.so
 $(LUA_MODULE_NAME): .make-prerequisites
-	cd modules/lua && $(MAKE) OPTIMIZATION="$(OPTIMIZATION)"
+	$(MAKE) -C modules/lua OPTIMIZATION="$(OPTIMIZATION)" STATIC_LUA="$(STATIC_LUA)"
 
 # valkey-cli
 $(ENGINE_CLI_NAME): $(ENGINE_CLI_OBJ)

src/config.c

@@ -1617,6 +1617,7 @@ void rewriteConfigLoadmoduleOption(struct rewriteConfigState *state) {
     dictEntry *de;
     while ((de = dictNext(di)) != NULL) {
         struct ValkeyModule *module = dictGetVal(de);
+        if (module->is_static_module) continue;
         line = moduleLoadQueueEntryToLoadmoduleOptionStr(module, "loadmodule");
         rewriteConfigRewriteLine(state, "loadmodule", line, 1);
     }

src/module.c

@@ -55,18 +55,17 @@
  * replacements are done, such as the replacement of RM with ValkeyModule in
  * function names. For details, see the script src/modules/gendoc.rb.
  * -------------------------------------------------------------------------- */
-
 #include "server.h"
 #include "cluster.h"
 #include "commandlog.h"
 #include "rdb.h"
 #include "monotonic.h"
 #include "script.h"
-#include "call_reply.h"
 #include "hdr_histogram.h"
 #include "crc16_slottable.h"
 #include "valkeymodule.h"
 #include "module.h"
+#include "call_reply.h"
 #include "io_threads.h"
 #include "scripting_engine.h"
 #include "cluster_migrateslots.h"
@@ -12889,6 +12888,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
     dictAdd(modules, ctx.module->name, ctx.module);
     ctx.module->blocked_clients = 0;
     ctx.module->handle = handle;
+    ctx.module->is_static_module = 0;
     ctx.module->loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry));
     ctx.module->loadmod->path = sdsnew(path);
     ctx.module->loadmod->argv = module_argc ? zmalloc(sizeof(robj *) * module_argc) : NULL;
@@ -12932,6 +12932,162 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
     return C_OK;
 }
 
+/**
+ * Load a static symbol from the current process by constructing a module-specific symbol name.
+ *
+ * This function builds the lookup name as "<symbol_name>_<module_name>", opens the current
+ * process image with dlopen(), and then resolves the symbol with dlsym(). On failure, it logs
+ * a warning and returns C_ERR; on success, it stores the resolved address and handle and returns
+ * C_OK.
+ *
+ * @param out
+ *   void **. Output pointer that receives the address of the resolved symbol.
+ * @param handle
+ *   void **. Output pointer that receives the dlopen() handle for the current process.
+ * @param symbol_name
+ *   const char *. Base name of the symbol to resolve.
+ * @param module_name
+ *   const char *. Module name appended to the symbol name to form the final lookup name.
+ *
+ * @return int
+ *   C_OK on success, or C_ERR if the symbol name is too long, dlopen() fails, or dlsym()
+ *   cannot resolve the symbol.
+ *
+ * @note This function is intended for use in the module-loading context and relies on the
+ * current process exposing statically linked module entry points under the constructed name.
+ */
+static int moduleLoadStaticSymbol(void **out, void **handle, const char *symbol_name, const char *module_name) {
+    char symbol_full_name[128];
+    int n = snprintf(symbol_full_name, sizeof(symbol_full_name), "%s_%s", symbol_name, module_name);
+    if (n >= (int)sizeof(symbol_full_name)) {
+        serverLog(LL_WARNING, "Module name is too long");
+        return C_ERR;
+    }
+
+    // Open a handle to self
+    *handle = dlopen(NULL, RTLD_NOW);
+    if (*handle == NULL) {
+        char *error = dlerror();
+        if (error == NULL) error = "Unknown error";
+
+        serverLog(LL_WARNING, "Failed to load static module: %s. %s", module_name, error);
+        return C_ERR;
+    }
+
+    *out = dlsym(*handle, symbol_full_name);
+    if (*out == NULL) {
+        char *error = dlerror();
+        if (error == NULL) error = "Unknown error";
+
+        serverLog(LL_WARNING,
+                  "Failed to load static module: %s. Could not load method: %s. %s", module_name,
+                  symbol_full_name, error);
+        dlclose(*handle);
+        return C_ERR;
+    }
+    return C_OK;
+}
+
+/**
+ * @brief Loads a statically linked module by locating its on-load and on-unload entry points.
+ *
+ * This function builds the expected symbol names for the module, resolves them from the current
+ * process image, and invokes the module's initialization routine. On success, the module is registered
+ * with the server, its state is initialized, and module load events are fired. If validation or
+ * post-load checks fail, the module is unloaded and an error is returned.
+ *
+ * @param module_name const char * Name of the static module, used to construct the on-load and
+ * on-unload symbol names.
+ * @param module_argv void ** Array of module arguments passed through to the module's on-load
+ * function.
+ * @param module_argc int Number of entries in module_argv.
+ * @param is_loadex int Nonzero if the module is being loaded via the loadex path and queued
+ * configuration arguments should be validated.
+ *
+ * @return int C_OK on successful load; C_ERR if symbol lookup fails, module initialization fails,
+ * or post-load validation detects an error.
+ *
+ * @note This function operates in the server/module loading context and expects the module's
+ * ValkeyModule_OnLoad_<name> and ValkeyModule_OnUnload_<name> symbols to be present in the
+ * current process.
+ *
+ * @throws No exceptions are thrown. Errors are reported via server logging and by returning C_ERR.
+ */
+int moduleLoadStatic(const char *module_name, void **module_argv, int module_argc, int is_loadex) {
+    // Locate the load
+    ModuleLoadFunc onload;
+    void *handle = NULL;
+    if (moduleLoadStaticSymbol((void **)&onload, &handle, "ValkeyModule_OnLoad", module_name) != C_OK) {
+        return C_ERR;
+    }
+
+    ValkeyModuleCtx ctx;
+    moduleCreateContext(&ctx, NULL, VALKEYMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */
+    if (onload((void *)&ctx, module_argv, module_argc) == VALKEYMODULE_ERR) {
+        if (ctx.module) {
+            serverLog(LL_WARNING, "Static Module %s initialization failed. Module not loaded.", module_name);
+            moduleUnregisterCleanup(ctx.module);
+            moduleRemoveCateogires(ctx.module);
+            moduleFreeModuleStructure(ctx.module);
+        } else {
+            /* If there is no ctx.module, this means that our ValkeyModule_Init call failed,
+             * and currently init will only fail on busy name. */
+            serverLog(LL_WARNING, "Static Module %s initialization failed. Module name is busy.", module_name);
+        }
+        moduleFreeContext(&ctx);
+        dlclose(handle);
+        return C_ERR;
+    }
+
+    dlclose(handle);
+
+    /* Module loaded! Register it. */
+    dictAdd(modules, ctx.module->name, ctx.module);
+    ctx.module->blocked_clients = 0;
+    ctx.module->handle = NULL; // We will re-open when needed.
+    ctx.module->is_static_module = 1;
+    ctx.module->loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry));
+    ctx.module->loadmod->path = sdsnew(module_name);
+    ctx.module->loadmod->argv = module_argc ? zmalloc(sizeof(robj *) * module_argc) : NULL;
+    ctx.module->loadmod->argc = module_argc;
+    for (int i = 0; i < module_argc; i++) {
+        ctx.module->loadmod->argv[i] = module_argv[i];
+        incrRefCount(ctx.module->loadmod->argv[i]);
+    }
+
+    /* If module commands have ACL categories, recompute command bits
+     * for all existing users once the modules has been registered. */
+    if (ctx.module->num_commands_with_acl_categories) {
+        ACLRecomputeCommandBitsFromCommandRulesAllUsers();
+    }
+    serverLog(LL_NOTICE, "Static Module '%s' successfully loaded", ctx.module->name);
+    ctx.module->onload = 0;
+
+    int post_load_err = 0;
+    if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) {
+        serverLogRaw(LL_WARNING,
+                     "Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module.");
+        post_load_err = 1;
+    }
+
+    if (is_loadex && dictSize(server.module_configs_queue)) {
+        serverLogRaw(LL_WARNING,
+                     "Loadex configurations were not applied, likely due to invalid arguments. Unloading the module.");
+        post_load_err = 1;
+    }
+
+    if (post_load_err) {
+        moduleUnload(ctx.module->name, NULL);
+        moduleFreeContext(&ctx);
+        return C_ERR;
+    }
+
+    /* Fire the loaded modules event. */
+    moduleFireServerEvent(VALKEYMODULE_EVENT_MODULE_CHANGE, VALKEYMODULE_SUBEVENT_MODULE_LOADED, ctx.module);
+    moduleFreeContext(&ctx);
+    return C_OK;
+}
+
 static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg) {
     if (listLength(module->types)) {
         *errmsg = "the module exports one or more module-side data "
@@ -12964,15 +13120,21 @@ static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg
     }
 
     /* Give module a chance to clean up. */
-    const char *onUnloadNames[] = {"ValkeyModule_OnUnload", "RedisModule_OnUnload"};
-    int (*onunload)(void *) = NULL;
-    for (size_t i = 0; i < sizeof(onUnloadNames) / sizeof(onUnloadNames[0]); i++) {
-        onunload = (int (*)(void *))(unsigned long)dlsym(module->handle, onUnloadNames[i]);
-        if (onunload) {
-            if (i != 0) {
-                serverLog(LL_NOTICE, "Legacy Redis Module %s found", module->name);
+    ModuleUnLoadFunc onunload = NULL;
+    if (module->is_static_module == 1) {
+        if (moduleLoadStaticSymbol((void **)&onunload, &module->handle, "ValkeyModule_OnUnload", module->name) != C_OK) {
+            return C_ERR;
+        }
+    } else {
+        const char *onUnloadNames[] = {"ValkeyModule_OnUnload", "RedisModule_OnUnload"};
+        for (size_t i = 0; i < sizeof(onUnloadNames) / sizeof(onUnloadNames[0]); i++) {
+            onunload = (int (*)(void *))(unsigned long)dlsym(module->handle, onUnloadNames[i]);
+            if (onunload) {
+                if (i != 0) {
+                    serverLog(LL_NOTICE, "Legacy Redis Module %s found", module->name);
+                }
+                break;
             }
-            break;
         }
     }
 

src/module.h

@@ -95,6 +95,9 @@ typedef struct moduleValue {
     void *value;
 } moduleValue;
 
+typedef int (*ModuleLoadFunc)(void *, void **, int);
+typedef int (*ModuleUnLoadFunc)(void *);
+
 /* This structure represents a module inside the system. */
 typedef struct ValkeyModule {
     void *handle;                         /* Module dlopen() handle. */
@@ -117,6 +120,7 @@ typedef struct ValkeyModule {
     int num_commands_with_acl_categories; /* Number of commands in this module included in acl categories */
     int onload;                           /* Flag to identify if the call is being made from Onload (0 or 1) */
     size_t num_acl_categories_added;      /* Number of acl categories added by this module. */
+    int is_static_module;                 /* 1 if this is a static module, 0 otherwise */
 } ValkeyModule;
 
 /* This is a wrapper for the 'rio' streams used inside rdb.c in the server, so that
@@ -182,6 +186,7 @@ void moduleInitModulesSystem(void);
 void moduleInitModulesSystemLast(void);
 void modulesCron(void);
 int moduleLoad(const char *path, void **argv, int argc, int is_loadex);
+int moduleLoadStatic(const char *path, void **argv, int argc, int is_loadex);
 int moduleUnload(sds name, const char **errmsg);
 void moduleUnloadAllModules(void);
 void moduleLoadFromQueue(void);