Streaming Compression support for RDB

cmake/Modules/SourceFiles.cmake

@@ -121,7 +121,11 @@ set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/vset.c
     ${CMAKE_SOURCE_DIR}/src/fifo.c
     ${CMAKE_SOURCE_DIR}/src/mutexqueue.c
-    ${CMAKE_SOURCE_DIR}/src/queues.c)
+    ${CMAKE_SOURCE_DIR}/src/queues.c
+    ${CMAKE_SOURCE_DIR}/src/compression.c
+    ${CMAKE_SOURCE_DIR}/src/compression_lz4.c
+    ${CMAKE_SOURCE_DIR}/src/compression_stream.c
+    ${CMAKE_SOURCE_DIR}/src/compression_rio.c)
 
 
 # valkey-cli

cmake/Modules/ValkeySetup.cmake

@@ -282,6 +282,7 @@ include_directories("${CMAKE_SOURCE_DIR}/src/modules/lua")
 include_directories("${CMAKE_SOURCE_DIR}/deps/linenoise")
 include_directories("${CMAKE_SOURCE_DIR}/deps/hdr_histogram")
 include_directories("${CMAKE_SOURCE_DIR}/deps/fpconv")
+include_directories("${CMAKE_SOURCE_DIR}/deps/lz4")
 
 add_subdirectory("${CMAKE_SOURCE_DIR}/deps")
 

deps/CMakeLists.txt

@@ -33,6 +33,7 @@ add_subdirectory(linenoise)
 add_subdirectory(fpconv)
 add_subdirectory(hdr_histogram)
 add_subdirectory(fast_float)
+add_subdirectory(lz4)
 
 # Clear any cached variables passed to libvalkey from the cache
 unset(BUILD_SHARED_LIBS CACHE)

deps/Makefile

@@ -42,6 +42,7 @@ distclean:
 	-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
 	-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
 	-(cd fpconv && $(MAKE) clean) > /dev/null || true
+	-(cd lz4 && $(MAKE) clean) > /dev/null || true
 	-(rm -f .make-*)
 
 .PHONY: distclean
@@ -132,3 +133,11 @@ gtest-parallel: .make-prerequisites
 		rm -rf gtest-parallel; \
 		git clone --depth 1 https://github.com/google/gtest-parallel.git gtest-parallel; \
 	fi
+
+.PHONY: gtest-parallel
+
+lz4: .make-prerequisites
+	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+	cd lz4 && $(MAKE) CFLAGS="$(CFLAGS)"
+
+.PHONY: lz4

deps/lz4/CMakeLists.txt

@@ -0,0 +1,14 @@
+project(lz4)
+
+set(SRCS
+    "${CMAKE_CURRENT_LIST_DIR}/lz4.c"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4.h"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4hc.c"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4hc.h"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4frame.c"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4frame.h"
+    "${CMAKE_CURRENT_LIST_DIR}/xxhash.c"
+    "${CMAKE_CURRENT_LIST_DIR}/xxhash.h")
+
+add_library(lz4 STATIC ${SRCS})
+set_target_properties(lz4 PROPERTIES POSITION_INDEPENDENT_CODE ON)

deps/lz4/LICENSE

@@ -0,0 +1,24 @@
+LZ4 Library
+Copyright (c) 2011-2020, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

deps/lz4/Makefile

@@ -0,0 +1,28 @@
+STD=
+WARN= -Wall
+OPT= -O3
+
+R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS)
+R_LDFLAGS= $(LDFLAGS)
+DEBUG= -g
+
+R_CC=$(CC) $(R_CFLAGS)
+R_LD=$(CC) $(R_LDFLAGS)
+
+AR= ar
+ARFLAGS= rcs
+
+liblz4.a: lz4.o lz4hc.o lz4frame.o xxhash.o
+	$(AR) $(ARFLAGS) $@ $^
+
+lz4.o: lz4.c lz4.h
+lz4hc.o: lz4hc.c lz4hc.h lz4.h
+lz4frame.o: lz4frame.c lz4frame.h lz4.h lz4hc.h xxhash.h
+xxhash.o: xxhash.c xxhash.h
+
+.c.o:
+	$(R_CC) -fPIC -c $<
+
+clean:
+	rm -f *.o
+	rm -f *.a