diff --git a/CMakeLists.txt b/CMakeLists.txt index 4aa26cfff8e..3c7c1b8a352 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ if (APPLE) endif () # Options -option(BUILD_LUA "Build Valkey Lua scripting engine" ON) +set(BUILD_LUA "static" CACHE STRING "Build Valkey Lua scripting engine: static (default), module, no") option(BUILD_UNIT_GTESTS "Build valkey-unit-gtests" OFF) option(BUILD_TEST_MODULES "Build all test modules" OFF) option(BUILD_EXAMPLE_MODULES "Build example modules" OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 98c963153a0..6ef9d816fcb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,24 +17,34 @@ add_dependencies(valkey-server generate_commands_def) add_dependencies(valkey-server generate_fmtargs_h) add_dependencies(valkey-server release_header) -if (BUILD_LUA) +if (NOT BUILD_LUA STREQUAL "no") message(STATUS "Build Lua scripting engine module") + if (BUILD_LUA STREQUAL "static") + add_compile_definitions(STATIC_LUA=1) + message(STATUS "Building LUA as a STATIC module") + else () + add_compile_definitions(STATIC_LUA=0) + message(STATUS "Building LUA as a DYNAMIC module") + endif () add_subdirectory(modules/lua) - add_dependencies(valkey-server valkeylua) + if (BUILD_LUA STREQUAL "static") + target_link_libraries(valkey-server $) + else () + add_dependencies(valkey-server valkeylua) + endif () target_compile_definitions(valkey-server PRIVATE LUA_ENABLED) if (UNIX AND NOT APPLE) target_compile_definitions(valkey-server PRIVATE LUA_LIB=libvalkeylua.so) - target_link_options(valkey-server PRIVATE -Wl,--disable-new-dtags) else () target_compile_definitions(valkey-server PRIVATE LUA_LIB=libvalkeylua.dylib) endif () set(VALKEY_INSTALL_RPATH "") - set_target_properties(valkey-server PROPERTIES - INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" - INSTALL_RPATH_USE_LINK_PATH TRUE - BUILD_WITH_INSTALL_RPATH TRUE - ) -endif() + set_target_properties( + valkey-server + PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" + INSTALL_RPATH_USE_LINK_PATH TRUE + BUILD_WITH_INSTALL_RPATH TRUE) +endif () unset(BUILD_LUA CACHE) if (VALKEY_RELEASE_BUILD) @@ -49,9 +59,8 @@ if (DEBUG_FORCE_DEFRAG) endif () if (BUILD_SANITIZER) - # 'BUILD_SANITIZER' is defined in ValkeySetup module (based on user input) - # If defined, the variables 'VALKEY_SANITAIZER_CFLAGS' and 'VALKEY_SANITAIZER_LDFLAGS' - # are set with the link & compile flags required + # 'BUILD_SANITIZER' is defined in ValkeySetup module (based on user input) If defined, the variables + # 'VALKEY_SANITAIZER_CFLAGS' and 'VALKEY_SANITAIZER_LDFLAGS' are set with the link & compile flags required message(STATUS "Adding sanitizer flags for target valkey-server") target_compile_options(valkey-server PRIVATE ${VALKEY_SANITAIZER_CFLAGS}) target_link_options(valkey-server PRIVATE ${VALKEY_SANITAIZER_LDFLAGS}) @@ -118,10 +127,10 @@ endif () # Friendly hint like the Makefile one file(RELATIVE_PATH _CMAKE_DIR_RELATIVE_PATH "${CMAKE_SOURCE_DIR}" "${CMAKE_BINARY_DIR}") -add_custom_target(hint ALL +add_custom_target( + hint ALL DEPENDS valkey-server valkey-cli valkey-benchmark COMMAND ${CMAKE_COMMAND} -E echo "" COMMAND ${CMAKE_COMMAND} -E echo "Hint: It is a good idea to run tests with your CMake-built binaries \\;\\)" COMMAND ${CMAKE_COMMAND} -E echo " ./${_CMAKE_DIR_RELATIVE_PATH}/runtest" - COMMAND ${CMAKE_COMMAND} -E echo "" -) + COMMAND ${CMAKE_COMMAND} -E echo "") diff --git a/src/Makefile b/src/Makefile index bcbf20a48ba..0560c13d988 100644 --- a/src/Makefile +++ b/src/Makefile @@ -259,22 +259,37 @@ endif FINAL_CFLAGS+= -I../deps/libvalkey/include -I../deps/linenoise -I../deps/hdr_histogram -I../deps/fpconv -I../deps/fast_float # Lua scripting engine module -LUA_MODULE_NAME:=modules/lua/libvalkeylua.so ifeq ($(BUILD_LUA),no) + LUA_MODULE_NAME= LUA_MODULE= LUA_MODULE_INSTALL= -else + FINAL_CFLAGS+=-DSTATIC_LUA=0 +else ifeq ($(BUILD_LUA),module) + LUA_MODULE_NAME=modules/lua/libvalkeylua.so LUA_MODULE=$(LUA_MODULE_NAME) LUA_MODULE_INSTALL=install-lua-module current_dir = $(shell pwd) - FINAL_CFLAGS+=-DLUA_ENABLED -DLUA_LIB=libvalkeylua.so -ifeq ($(uname_S),Darwin) - FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib - FINAL_LDFLAGS+= -Wl,-rpath,$(current_dir)/modules/lua + FINAL_CFLAGS+=-DLUA_ENABLED -DLUA_LIB=libvalkeylua.so -DSTATIC_LUA=0 + ifeq ($(uname_S),Darwin) + FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib + FINAL_LDFLAGS+= -Wl,-rpath,$(current_dir)/modules/lua + else + FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib:$(current_dir)/modules/lua -Wl,--disable-new-dtags + endif else - FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib:$(current_dir)/modules/lua -Wl,--disable-new-dtags -endif + # The default: building Lua as a static module. + LUA_MODULE_NAME=modules/lua/libvalkeylua.a + LUA_MODULE=$(LUA_MODULE_NAME) + current_dir = $(shell pwd) + FINAL_CFLAGS+=-DLUA_ENABLED -DSTATIC_LUA=1 + ifeq ($(uname_S),Darwin) + LUA_LDFLAGS=-Wl,-export_dynamic -Wl,-force_load,$(current_dir)/modules/lua/libvalkeylua.a ../deps/lua/src/liblua.a + else ifeq ($(uname_S),FreeBSD) + LUA_LDFLAGS=-Wl,--export-dynamic -Wl,--whole-archive $(current_dir)/modules/lua/libvalkeylua.a -Wl,--no-whole-archive ../deps/lua/src/liblua.a + else + LUA_LDFLAGS=-Wl,--whole-archive $(current_dir)/modules/lua/libvalkeylua.a -Wl,--no-whole-archive ../deps/lua/src/liblua.a + endif endif # Determine systemd support and/or build preference (defaulting to auto-detection) @@ -693,7 +708,7 @@ endif # valkey-server $(SERVER_NAME): $(ENGINE_SERVER_OBJ) $(LUA_MODULE) - $(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) + $(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(LUA_LDFLAGS) # Valkey static library, used to compile against for unit testing $(ENGINE_LIB_NAME): $(ENGINE_SERVER_OBJ) @@ -721,7 +736,7 @@ $(RDMA_MODULE_NAME): $(SERVER_NAME) # engine_lua.so $(LUA_MODULE_NAME): .make-prerequisites - cd modules/lua && $(MAKE) OPTIMIZATION="$(OPTIMIZATION)" + $(MAKE) -C modules/lua OPTIMIZATION="$(OPTIMIZATION)" BUILD_LUA="$(BUILD_LUA)" # valkey-cli $(ENGINE_CLI_NAME): $(ENGINE_CLI_OBJ) diff --git a/src/config.c b/src/config.c index 3e7f5a0ea67..8cd049e69ed 100644 --- a/src/config.c +++ b/src/config.c @@ -1619,6 +1619,7 @@ void rewriteConfigLoadmoduleOption(struct rewriteConfigState *state) { dictEntry *de; while ((de = dictNext(di)) != NULL) { struct ValkeyModule *module = dictGetVal(de); + if (module->is_static_module) continue; line = moduleLoadQueueEntryToLoadmoduleOptionStr(module, "loadmodule"); rewriteConfigRewriteLine(state, "loadmodule", line, 1); } diff --git a/src/eval.c b/src/eval.c index 5f56f782fa1..ea2bb98c133 100644 --- a/src/eval.c +++ b/src/eval.c @@ -181,7 +181,6 @@ void evalRelease(int async) { } } - void evalReset(int async) { evalRelease(async); evalInit(); diff --git a/src/module.c b/src/module.c index adc8870ede7..79d20e0ffb0 100644 --- a/src/module.c +++ b/src/module.c @@ -55,18 +55,17 @@ * replacements are done, such as the replacement of RM with ValkeyModule in * function names. For details, see the script src/modules/gendoc.rb. * -------------------------------------------------------------------------- */ - #include "server.h" #include "cluster.h" #include "commandlog.h" #include "rdb.h" #include "monotonic.h" #include "script.h" -#include "call_reply.h" #include "hdr_histogram.h" #include "crc16_slottable.h" #include "valkeymodule.h" #include "module.h" +#include "call_reply.h" #include "io_threads.h" #include "scripting_engine.h" #include "cluster_migrateslots.h" @@ -13075,10 +13074,100 @@ void moduleUnregisterCleanup(ValkeyModule *module) { moduleUnregisterAuthCBs(module); } +/* Common helper for moduleLoad and moduleLoadStatic. + * Invokes the onload callback, registers the module, and performs post-load + * validation. 'display_name' is used in log messages, 'handle' is the + * dlopen handle (NULL for static modules), and 'is_static' controls the + * is_static_module flag and handle ownership semantics. */ +static int moduleInitPostOnLoadResolved(ModuleLoadFunc onload, + void *handle, + const char *display_name, + void **module_argv, + int module_argc, + int is_loadex, + int is_static) { + ValkeyModuleCtx ctx; + moduleCreateContext(&ctx, NULL, VALKEYMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */ + if (onload((void *)&ctx, module_argv, module_argc) == VALKEYMODULE_ERR) { + if (ctx.module) { + serverLog(LL_WARNING, "%sModule %s initialization failed. Module not loaded.", + is_static ? "Static " : "", display_name); + moduleUnregisterCleanup(ctx.module); + moduleRemoveCateogires(ctx.module); + moduleFreeModuleStructure(ctx.module); + } else { + /* If there is no ctx.module, this means that our ValkeyModule_Init call failed, + * and currently init will only fail on busy name. */ + serverLog(LL_WARNING, "%sModule %s initialization failed. Module name is busy.", + is_static ? "Static " : "", display_name); + } + moduleFreeContext(&ctx); + if (handle) { + dlclose(handle); + } + return C_ERR; + } + + if (is_static && handle) { + dlclose(handle); + handle = NULL; + } + + /* Module loaded! Register it. */ + dictAdd(modules, ctx.module->name, ctx.module); + ctx.module->blocked_clients = 0; + ctx.module->handle = handle; + ctx.module->is_static_module = is_static; + ctx.module->loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry)); + ctx.module->loadmod->path = sdsnew(display_name); + ctx.module->loadmod->argv = module_argc ? zmalloc(sizeof(robj *) * module_argc) : NULL; + ctx.module->loadmod->argc = module_argc; + for (int i = 0; i < module_argc; i++) { + ctx.module->loadmod->argv[i] = module_argv[i]; + incrRefCount(ctx.module->loadmod->argv[i]); + } + + /* If module commands have ACL categories, recompute command bits + * for all existing users once the modules has been registered. */ + if (ctx.module->num_commands_with_acl_categories) { + ACLRecomputeCommandBitsFromCommandRulesAllUsers(); + } + if (is_static) { + serverLog(LL_NOTICE, "Static Module '%s' successfully loaded", ctx.module->name); + } else { + serverLog(LL_NOTICE, "Module '%s' loaded from %s", ctx.module->name, display_name); + } + ctx.module->onload = 0; + + int post_load_err = 0; + if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) { + serverLogRaw(LL_WARNING, + "Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module."); + post_load_err = 1; + } + + if (is_loadex && dictSize(server.module_configs_queue)) { + serverLogRaw(LL_WARNING, + "Loadex configurations were not applied, likely due to invalid arguments. Unloading the module."); + post_load_err = 1; + } + + if (post_load_err) { + moduleUnload(ctx.module->name, NULL); + moduleFreeContext(&ctx); + return C_ERR; + } + + /* Fire the loaded modules event. */ + moduleFireServerEvent(VALKEYMODULE_EVENT_MODULE_CHANGE, VALKEYMODULE_SUBEVENT_MODULE_LOADED, ctx.module); + moduleFreeContext(&ctx); + return C_OK; +} + /* Load a module and initialize it. On success C_OK is returned, otherwise * C_ERR is returned. */ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loadex) { - int (*onload)(void *, void **, int); + ModuleLoadFunc onload; void *handle; struct stat st; @@ -13109,7 +13198,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa const char *onLoadNames[] = {"ValkeyModule_OnLoad", "RedisModule_OnLoad"}; for (size_t i = 0; i < sizeof(onLoadNames) / sizeof(onLoadNames[0]); i++) { - onload = (int (*)(void *, void **, int))(unsigned long)dlsym(handle, onLoadNames[i]); + onload = (ModuleLoadFunc)(unsigned long)dlsym(handle, onLoadNames[i]); if (onload != NULL) { if (i != 0) { serverLog(LL_NOTICE, "Legacy Redis Module %s found", path); @@ -13126,69 +13215,73 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa path); return C_ERR; } - ValkeyModuleCtx ctx; - moduleCreateContext(&ctx, NULL, VALKEYMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */ - if (onload((void *)&ctx, module_argv, module_argc) == VALKEYMODULE_ERR) { - if (ctx.module) { - serverLog(LL_WARNING, "Module %s initialization failed. Module not loaded.", path); - moduleUnregisterCleanup(ctx.module); - moduleRemoveCateogires(ctx.module); - moduleFreeModuleStructure(ctx.module); - } else { - /* If there is no ctx.module, this means that our ValkeyModule_Init call failed, - * and currently init will only fail on busy name. */ - serverLog(LL_WARNING, "Module %s initialization failed. Module name is busy.", path); - } - moduleFreeContext(&ctx); - dlclose(handle); + return moduleInitPostOnLoadResolved(onload, handle, path, module_argv, module_argc, is_loadex, 0); +} + +/* Resolve a symbol from a statically linked module. The symbol is looked up + * by constructing the name "_" and searching for it + * in the current process via dlopen(NULL)/dlsym(). On success, '*out' is set + * to the symbol address, '*handle' is set to the dlopen handle, and C_OK is + * returned. On failure C_ERR is returned and an appropriate warning is logged. */ +static int moduleLoadStaticSymbol(void **out, void **handle, const char *symbol_name, const char *module_name) { + char symbol_full_name[128]; + int n = snprintf(symbol_full_name, sizeof(symbol_full_name), "%s_%s", symbol_name, module_name); + if (n >= (int)sizeof(symbol_full_name)) { + serverLog(LL_WARNING, "Module name is too long"); return C_ERR; } - /* Module loaded! Register it. */ - dictAdd(modules, ctx.module->name, ctx.module); - ctx.module->blocked_clients = 0; - ctx.module->handle = handle; - ctx.module->loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry)); - ctx.module->loadmod->path = sdsnew(path); - ctx.module->loadmod->argv = module_argc ? zmalloc(sizeof(robj *) * module_argc) : NULL; - ctx.module->loadmod->argc = module_argc; - for (int i = 0; i < module_argc; i++) { - ctx.module->loadmod->argv[i] = module_argv[i]; - incrRefCount(ctx.module->loadmod->argv[i]); - } + /* Open a handle to self */ + *handle = dlopen(NULL, RTLD_NOW); + if (*handle == NULL) { + char *error = dlerror(); + if (error == NULL) error = "Unknown error"; - /* If module commands have ACL categories, recompute command bits - * for all existing users once the modules has been registered. */ - if (ctx.module->num_commands_with_acl_categories) { - ACLRecomputeCommandBitsFromCommandRulesAllUsers(); + serverLog(LL_WARNING, "Failed to load static module: %s. %s", module_name, error); + return C_ERR; } - serverLog(LL_NOTICE, "Module '%s' loaded from %s", ctx.module->name, path); - ctx.module->onload = 0; - int post_load_err = 0; - if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) { - serverLogRaw(LL_WARNING, - "Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module."); - post_load_err = 1; - } + *out = dlsym(*handle, symbol_full_name); + if (*out == NULL) { + char *error = dlerror(); + if (error == NULL) error = "Unknown error"; - if (is_loadex && dictSize(server.module_configs_queue)) { - serverLogRaw(LL_WARNING, - "Loadex configurations were not applied, likely due to invalid arguments. Unloading the module."); - post_load_err = 1; + serverLog(LL_WARNING, + "Failed to load static module: %s. Could not load method: %s. %s", module_name, + symbol_full_name, error); + dlclose(*handle); + return C_ERR; } + return C_OK; +} - if (post_load_err) { - moduleUnload(ctx.module->name, NULL); - moduleFreeContext(&ctx); +/* Load a statically linked module and initialize it. This is the static + * counterpart of moduleLoad(): instead of dlopen()ing a shared object from + * a file path, it resolves the module's entry point from the running + * executable itself. + * + * The entry point is located by constructing the symbol name + * "ValkeyModule_OnLoad_" and resolving it with + * moduleLoadStaticSymbol(). For example, a module named "mymodule" must + * provide a function called ValkeyModule_OnLoad_mymodule. + * + * Once the entry point is found, the function creates a temporary module + * context, invokes the OnLoad callback, and on success registers the module + * in the global modules dictionary with is_static_module set to 1 and handle + * set to NULL (since there is no shared-object handle to keep open). + * + * If 'is_loadex' is true, the function also validates that all queued module + * configurations were consumed; otherwise the module is unloaded. + * + * On success C_OK is returned, otherwise C_ERR is returned. */ +int moduleLoadStatic(const char *module_name, void **module_argv, int module_argc, int is_loadex) { + ModuleLoadFunc onload; + void *handle = NULL; + if (moduleLoadStaticSymbol((void **)&onload, &handle, "ValkeyModule_OnLoad", module_name) != C_OK) { return C_ERR; } - - /* Fire the loaded modules event. */ - moduleFireServerEvent(VALKEYMODULE_EVENT_MODULE_CHANGE, VALKEYMODULE_SUBEVENT_MODULE_LOADED, ctx.module); - - moduleFreeContext(&ctx); - return C_OK; + return moduleInitPostOnLoadResolved(onload, handle, module_name, module_argv, module_argc, + is_loadex, 1); } static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg) { @@ -13223,15 +13316,23 @@ static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg } /* Give module a chance to clean up. */ - const char *onUnloadNames[] = {"ValkeyModule_OnUnload", "RedisModule_OnUnload"}; - int (*onunload)(void *) = NULL; - for (size_t i = 0; i < sizeof(onUnloadNames) / sizeof(onUnloadNames[0]); i++) { - onunload = (int (*)(void *))(unsigned long)dlsym(module->handle, onUnloadNames[i]); - if (onunload) { - if (i != 0) { - serverLog(LL_NOTICE, "Legacy Redis Module %s found", module->name); + ModuleUnLoadFunc onunload = NULL; + if (module->is_static_module == 1) { + if (moduleLoadStaticSymbol((void **)&onunload, &module->handle, "ValkeyModule_OnUnload", module->name) != C_OK) { + serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", module->name); + errno = ECANCELED; + return C_ERR; + } + } else { + const char *onUnloadNames[] = {"ValkeyModule_OnUnload", "RedisModule_OnUnload"}; + for (size_t i = 0; i < sizeof(onUnloadNames) / sizeof(onUnloadNames[0]); i++) { + onunload = (int (*)(void *))(unsigned long)dlsym(module->handle, onUnloadNames[i]); + if (onunload) { + if (i != 0) { + serverLog(LL_NOTICE, "Legacy Redis Module %s found", module->name); + } + break; } - break; } } @@ -13251,7 +13352,7 @@ static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg moduleUnregisterCleanup(module); /* Unload the dynamic library. */ - if (dlclose(module->handle) == -1) { + if (module->handle != NULL && dlclose(module->handle) == -1) { char *error = dlerror(); if (error == NULL) error = "Unknown error"; serverLog(LL_WARNING, "Error when trying to close the %s module: %s", module->name, error); diff --git a/src/module.h b/src/module.h index c7ad384c6a1..91daac94c60 100644 --- a/src/module.h +++ b/src/module.h @@ -95,6 +95,9 @@ typedef struct moduleValue { void *value; } moduleValue; +typedef int (*ModuleLoadFunc)(void *, void **, int); +typedef int (*ModuleUnLoadFunc)(void *); + /* This structure represents a module inside the system. */ typedef struct ValkeyModule { void *handle; /* Module dlopen() handle. */ @@ -117,6 +120,7 @@ typedef struct ValkeyModule { int num_commands_with_acl_categories; /* Number of commands in this module included in acl categories */ int onload; /* Flag to identify if the call is being made from Onload (0 or 1) */ size_t num_acl_categories_added; /* Number of acl categories added by this module. */ + int is_static_module; /* 1 if this is a static module, 0 otherwise */ } ValkeyModule; /* This is a wrapper for the 'rio' streams used inside rdb.c in the server, so that @@ -182,6 +186,7 @@ void moduleInitModulesSystem(void); void moduleInitModulesSystemLast(void); void modulesCron(void); int moduleLoad(const char *path, void **argv, int argc, int is_loadex); +int moduleLoadStatic(const char *path, void **argv, int argc, int is_loadex); int moduleUnload(sds name, const char **errmsg); void moduleUnloadAllModules(void); void moduleLoadFromQueue(void); diff --git a/src/modules/lua/CMakeLists.txt b/src/modules/lua/CMakeLists.txt index 00498d1a0b8..6b8d40a3921 100644 --- a/src/modules/lua/CMakeLists.txt +++ b/src/modules/lua/CMakeLists.txt @@ -4,22 +4,30 @@ if (VALKEY_DEBUG_BUILD) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W -Wall -fno-common -g -ggdb -std=c99 -O2 -D_GNU_SOURCE") else () set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W -Wall -fno-common -O3 -std=c99 -D_GNU_SOURCE") -endif() +endif () set(LUA_ENGINE_SRCS engine_lua.c script_lua.c function_lua.c debug_lua.c - list.c - ../../sha1.c - ../../rand.c) + list.c) -add_library(valkeylua SHARED "${LUA_ENGINE_SRCS}") +if (BUILD_LUA STREQUAL "module") + list(APPEND LUA_ENGINE_SRCS ../../sha1.c) + list(APPEND LUA_ENGINE_SRCS ../../rand.c) +endif () -target_link_libraries(valkeylua PRIVATE lualib fpconv) -target_include_directories(valkeylua PRIVATE ../../../deps/lua/src) +if (BUILD_LUA STREQUAL "static") + message(STATUS "Building STATIC LUA module") + add_library(valkeylua STATIC "${LUA_ENGINE_SRCS}") + target_link_libraries(valkeylua PUBLIC lualib fpconv) + target_include_directories(valkeylua PUBLIC ../../../deps/lua/src) +else () + message(STATUS "Building DYNAMIC LUA module") + add_library(valkeylua SHARED "${LUA_ENGINE_SRCS}") + target_link_libraries(valkeylua PRIVATE lualib fpconv) + target_include_directories(valkeylua PRIVATE ../../../deps/lua/src) +endif () -install(TARGETS valkeylua - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -) +install(TARGETS valkeylua LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/src/modules/lua/Makefile b/src/modules/lua/Makefile index 830461ddb4f..8f534a51b48 100644 --- a/src/modules/lua/Makefile +++ b/src/modules/lua/Makefile @@ -3,11 +3,19 @@ CLANG := $(findstring clang,$(shell sh -c '$(CC) --version | head -1')) DEPS_DIR=../../../deps +ifeq ($(BUILD_LUA),module) + STATIC_LUA_FLAG=-DSTATIC_LUA=0 + LUA_TARGET=libvalkeylua.so +else + STATIC_LUA_FLAG=-DSTATIC_LUA=1 + LUA_TARGET=libvalkeylua.a +endif + ifeq ($(uname_S),Darwin) - SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -dynamic -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(CFLAGS) + SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -dynamic -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(STATIC_LUA_FLAG) $(CFLAGS) SHOBJ_LDFLAGS= -bundle -undefined dynamic_lookup $(LDFLAGS) else - SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(CFLAGS) + SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(STATIC_LUA_FLAG) $(CFLAGS) SHOBJ_LDFLAGS= -shared $(LDFLAGS) # Pretty-printing setup for module build @@ -43,10 +51,15 @@ LIBS = \ $(DEPS_DIR)/lua/src/liblua.a \ $(DEPS_DIR)/fpconv/libfpconv.a SRCS= $(wildcard *.c) + +ifeq ($(BUILD_LUA),module) OBJS = \ - $(SRCS:.c=.o) \ - sha1.o \ - rand.o + $(SRCS:.c=.o) \ + sha1.o \ + rand.o +else +OBJS = $(SRCS:.c=.o) +endif # OS X 11.x doesn't have /usr/lib/libSystem.dylib and needs an explicit setting. ifeq ($(uname_S),Darwin) @@ -55,10 +68,14 @@ ifeq ("$(wildcard /usr/lib/libSystem.dylib)","") endif endif -all: libvalkeylua.so - -libvalkeylua.so: $(OBJS) $(LIBS) +all: $(LUA_TARGET) +ifeq ($(BUILD_LUA),module) +$(LUA_TARGET): $(OBJS) $(LIBS) $(QUIET_LINK_MOD)$(CC) -o $@ $(SHOBJ_LDFLAGS) $^ +else +$(LUA_TARGET): $(OBJS) $(LIBS) + $(QUIET_LINK_MOD)$(AR) rcs $@ $(OBJS) +endif sha1.o: ../../sha1.c $(QUIET_CC_MOD)$(CC) $(SHOBJ_CFLAGS) -c $< -o $@ @@ -76,4 +93,4 @@ $(DEPS_DIR)/fpconv/libfpconv.a: cd $(DEPS_DIR) && $(MAKE) fpconv clean: - rm -f *.so $(OBJS) + rm -f *.so *.a $(OBJS) diff --git a/src/modules/lua/engine_lua.c b/src/modules/lua/engine_lua.c index afdc4e20f32..91582481abc 100644 --- a/src/modules/lua/engine_lua.c +++ b/src/modules/lua/engine_lua.c @@ -480,9 +480,20 @@ static void luaEngineDebuggerEnd(ValkeyModuleCtx *module_ctx, static struct luaEngineCtx *engine_ctx = NULL; -int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, - ValkeyModuleString **argv, - int argc) { +#if STATIC_LUA +/* + * When building Lua as a static library, hide the generic module entry + * points, ValkeyModule_OnLoad and ValkeyModule_OnLoad, to avoid multiple + * symbol definitions. This is done by declaring these functions as static. + */ +#define LUA_MODULE_VISIBILITY static +#else +#define LUA_MODULE_VISIBILITY +#endif + +LUA_MODULE_VISIBILITY int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, + ValkeyModuleString **argv, + int argc) { VALKEYMODULE_NOT_USED(argv); VALKEYMODULE_NOT_USED(argc); @@ -533,7 +544,8 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, return VALKEYMODULE_OK; } -int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) { + +LUA_MODULE_VISIBILITY int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) { if (ValkeyModule_UnregisterScriptingEngine(ctx, LUA_ENGINE_NAME) != VALKEYMODULE_OK) { ValkeyModule_Log(ctx, "error", "Failed to unregister engine"); return VALKEYMODULE_ERR; @@ -544,3 +556,16 @@ int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) { return VALKEYMODULE_OK; } + +#if STATIC_LUA +/* Unique entry points (Load and Unload) used by the Lua module when linked statically */ +int ValkeyModule_OnLoad_lua(ValkeyModuleCtx *ctx, + ValkeyModuleString **argv, + int argc) { + return ValkeyModule_OnLoad(ctx, argv, argc); +} + +int ValkeyModule_OnUnload_lua(ValkeyModuleCtx *ctx) { + return ValkeyModule_OnUnload(ctx); +} +#endif diff --git a/src/modules/lua/function_lua.c b/src/modules/lua/function_lua.c index 3dd9b169d30..95283644a45 100644 --- a/src/modules/lua/function_lua.c +++ b/src/modules/lua/function_lua.c @@ -55,7 +55,11 @@ typedef uint64_t monotime; -static monotime getMonotonicUs(void) { +#if STATIC_LUA +/* Use the engine's version */ +#include "../../monotonic.h" +#else +monotime getMonotonicUs(void) { /* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems * did not support this until much later. CLOCK_MONOTONIC is technically * optional and may not be supported - but it appears to be universal. @@ -64,14 +68,14 @@ static monotime getMonotonicUs(void) { clock_gettime(CLOCK_MONOTONIC, &ts); return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000; } - -static inline uint64_t elapsedUs(monotime start_time) { +inline uint64_t elapsedUs(monotime start_time) { return getMonotonicUs() - start_time; } -static inline uint64_t elapsedMs(monotime start_time) { +inline uint64_t elapsedMs(monotime start_time) { return elapsedUs(start_time) / 1000; } +#endif typedef struct loadCtx { List *functions; @@ -213,7 +217,7 @@ typedef struct flagStr { const char *str; } flagStr; -flagStr scripts_flags_def[] = { +static flagStr lua_scripts_flags[] = { {.flag = VMSE_SCRIPT_FLAG_NO_WRITES, .str = "no-writes"}, {.flag = VMSE_SCRIPT_FLAG_ALLOW_OOM, .str = "allow-oom"}, {.flag = VMSE_SCRIPT_FLAG_ALLOW_STALE, .str = "allow-stale"}, @@ -244,7 +248,7 @@ static int luaRegisterFunctionReadFlags(lua_State *lua, uint64_t *flags) { const char *flag_str = lua_tostring(lua, -1); int found = 0; - for (flagStr *flag = scripts_flags_def; flag->str; ++flag) { + for (flagStr *flag = lua_scripts_flags; flag->str; ++flag) { if (!strcasecmp(flag->str, flag_str)) { f_flags |= flag->flag; found = 1; diff --git a/src/modules/lua/script_lua.c b/src/modules/lua/script_lua.c index c69a0f7c038..5786a3fbe44 100644 --- a/src/modules/lua/script_lua.c +++ b/src/modules/lua/script_lua.c @@ -176,26 +176,6 @@ static void _serverPanic(const char *file, int line, const char *msg, ...) { #define serverPanic(...) _serverPanic(__FILE__, __LINE__, __VA_ARGS__) -typedef uint64_t monotime; - -monotime getMonotonicUs(void) { - /* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems - * did not support this until much later. CLOCK_MONOTONIC is technically - * optional and may not be supported - but it appears to be universal. - * If this is not supported, provide a system-specific alternate version. */ - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000; -} - -inline uint64_t elapsedUs(monotime start_time) { - return getMonotonicUs() - start_time; -} - -inline uint64_t elapsedMs(monotime start_time) { - return elapsedUs(start_time) / 1000; -} - static int server_math_random(lua_State *L); static int server_math_randomseed(lua_State *L); @@ -1280,7 +1260,7 @@ static int luaRedisPCallCommand(lua_State *lua) { * * 'digest' should point to a 41 bytes buffer: 40 for SHA1 converted into an * hexadecimal number, plus 1 byte for null term. */ -void sha1hex(char *digest, char *script, size_t len) { +__attribute__((weak)) void sha1hex(char *digest, char *script, size_t len) { SHA1_CTX ctx; unsigned char hash[20]; char *cset = "0123456789abcdef"; diff --git a/src/server.c b/src/server.c index 03050bbae6e..506e7d90e42 100644 --- a/src/server.c +++ b/src/server.c @@ -7697,17 +7697,15 @@ __attribute__((weak)) int main(int argc, char **argv) { clusterInitLast(); } - /* Initialize the LUA scripting engine. */ -#ifdef LUA_ENABLED -#define LUA_LIB_STR STRINGIFY(LUA_LIB) +#if defined(LUA_ENABLED) && STATIC_LUA + /* Initialize the LUA scripting engine on-startup only when LUA is built statically */ if (scriptingEngineManagerFind("lua") == NULL) { - if (moduleLoad(LUA_LIB_STR, NULL, 0, 0) != C_OK) { + if (moduleLoadStatic("lua", NULL, 0, 0) != C_OK) { serverPanic("Lua engine initialization failed, check the server logs."); } } #endif - InitServerLast(); if (!server.sentinel_mode) {