From c9c77648c8b305a62790f4ffa8b9240e46d88310 Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Sun, 6 Jul 2025 21:07:46 +0530 Subject: [PATCH 01/13] feat: introduces Spicy runtime bridge and initial Spicy-HTTP parser * Adds C++ bridge (bridge.{h,cpp}) to embed Spicy/HILTI in Glutton * Spawns single-threaded worker goroutine for parser calls * Registers and exposes parsers via spicy.Initialize() * Implements Spicy-based HTTP handler and grammar (parsers/http.spicy) * Adds wire protocol selector in protocols.go (toggle with viper `spicy.enabled`) * Adds build rules for Spicy grammars (protocols/spicy/Makefile) --- .gitignore | 4 + Makefile | 5 +- config/config.yaml | 3 + glutton.go | 13 +- protocols/protocols.go | 8 +- protocols/spicy/Makefile | 30 +++ protocols/spicy/bridge.cpp | 308 +++++++++++++++++++++++++++++ protocols/spicy/bridge.h | 63 ++++++ protocols/spicy/handlers/http.go | 203 +++++++++++++++++++ protocols/spicy/helpers.go | 135 +++++++++++++ protocols/spicy/parser.go | 195 ++++++++++++++++++ protocols/spicy/parsers/http.spicy | 37 ++++ protocols/tcp/http.go | 2 +- protocols/tcp/resources.go | 2 +- 14 files changed, 1003 insertions(+), 5 deletions(-) create mode 100644 protocols/spicy/Makefile create mode 100644 protocols/spicy/bridge.cpp create mode 100644 protocols/spicy/bridge.h create mode 100644 protocols/spicy/handlers/http.go create mode 100644 protocols/spicy/helpers.go create mode 100644 protocols/spicy/parser.go create mode 100644 protocols/spicy/parsers/http.spicy diff --git a/.gitignore b/.gitignore index 16257c2..b432aaf 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ poc/ # Dev .vscode + +# Spicy generated files +protocols/spicy/*.cc +protocols/spicy/parsers/*.h diff --git a/Makefile b/Makefile index eedd1d5..2382ade 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,10 @@ upx: default: build build: - go build -ldflags=$(LDFLAGS) -o bin/server app/server.go + CC=clang CXX=clang++ go build -o bin/server app/server.go + +spicy: + cd protocols/spicy && make static: go build --ldflags '-extldflags "-static"' -o bin/server app/server.go diff --git a/config/config.yaml b/config/config.yaml index 222deb7..9e92e41 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -24,3 +24,6 @@ producers: conn_timeout: 45 max_tcp_payload: 4096 + +spicy: + enabled: true diff --git a/glutton.go b/glutton.go index c6ef637..fb5a62c 100644 --- a/glutton.go +++ b/glutton.go @@ -18,6 +18,7 @@ import ( "github.com/mushorg/glutton/connection" "github.com/mushorg/glutton/producer" "github.com/mushorg/glutton/protocols" + "github.com/mushorg/glutton/protocols/spicy" "github.com/mushorg/glutton/rules" "github.com/google/uuid" @@ -134,6 +135,11 @@ func (g *Glutton) Init() error { g.tcpProtocolHandlers = protocols.MapTCPProtocolHandlers(g.Logger, g) g.udpProtocolHandlers = protocols.MapUDPProtocolHandlers(g.Logger, g) + // Initializing Spicy parsers + if viper.GetBool("spicy.enabled") { + spicy.Initialize(g.Logger) + } + return nil } @@ -358,7 +364,12 @@ func (g *Glutton) Shutdown() { if err := flushTProxyIPTables(viper.GetString("interface"), g.publicAddrs[0].String(), "udp", uint32(g.Server.udpPort), uint32(viper.GetInt("ports.ssh"))); err != nil { g.Logger.Error("Failed to drop udp iptables", producer.ErrAttr(err)) } - + if viper.GetBool("spicy.enabled") { + g.Logger.Info("Cleaning up and shutting down Spicy and HILTI runtimes") + if err := spicy.Cleanup(); err != nil { + g.Logger.Error("Failed to clean up Spicy and HILTI runtimes", producer.ErrAttr(err)) + } + } g.Logger.Info("All done") } diff --git a/protocols/protocols.go b/protocols/protocols.go index d43b3ca..9a632d6 100644 --- a/protocols/protocols.go +++ b/protocols/protocols.go @@ -10,8 +10,10 @@ import ( "github.com/mushorg/glutton/connection" "github.com/mushorg/glutton/producer" "github.com/mushorg/glutton/protocols/interfaces" + spicyHandlers "github.com/mushorg/glutton/protocols/spicy/handlers" "github.com/mushorg/glutton/protocols/tcp" "github.com/mushorg/glutton/protocols/udp" + "github.com/spf13/viper" ) type TCPHandlerFunc func(ctx context.Context, conn net.Conn, md connection.Metadata) error @@ -84,7 +86,11 @@ func MapTCPProtocolHandlers(log interfaces.Logger, h interfaces.Honeypot) map[st // poor mans check for HTTP request httpMap := map[string]bool{"GET ": true, "POST": true, "HEAD": true, "OPTI": true, "CONN": true} if _, ok := httpMap[strings.ToUpper(string(snip))]; ok { - return tcp.HandleHTTP(ctx, bufConn, md, log, h) + if viper.GetBool("spicy.enabled") { + return spicyHandlers.HandleHTTP(ctx, bufConn, md, log, h) + } else { + return tcp.HandleHTTP(ctx, bufConn, md, log, h) + } } // poor mans check for RDP header if bytes.Equal(snip, []byte{0x03, 0x00, 0x00, 0x2b}) { diff --git a/protocols/spicy/Makefile b/protocols/spicy/Makefile new file mode 100644 index 0000000..985aed7 --- /dev/null +++ b/protocols/spicy/Makefile @@ -0,0 +1,30 @@ +GRAMMARS := $(wildcard parsers/*.spicy) + +GEN_CC := $(notdir $(GRAMMARS:.spicy=.cc)) +LINKER_CC := $(notdir $(patsubst parsers/%.spicy,spicy_linker_%.cc,$(GRAMMARS))) +HEADERS := $(patsubst parsers/%.spicy,parsers/%.h,$(GRAMMARS)) + +SPICY_FLAGS := -g +CXX ?= clang++ +CXXFLAGS += -I/opt/spicy/include -std=c++17 -fPIC -O3 -DNDEBUG -fvisibility=hidden -I$(CURDIR)/parsers + +.SECONDARY: $(GEN_CC) $(LINKER_CC) $(HEADERS) + +%.cc: parsers/%.spicy + @echo "spicyc -c $< -> $@" + spicyc -c $(SPICY_FLAGS) $< -o $@ + +parsers/%.h: parsers/%.spicy + @echo "spicyc -P parsers/$* -o $@ $<" + spicyc -P parsers/$* -o $@ $< + +spicy_linker_%.cc: parsers/%.spicy %.cc + @echo "spicyc -l $< -> $@" + spicyc -l $(SPICY_FLAGS) $< -o $@ + +.PHONY: all +all: $(GEN_CC) $(LINKER_CC) $(HEADERS) + +.PHONY: clean +clean: + rm -f $(GEN_CC) $(LINKER_CC) $(HEADERS) \ No newline at end of file diff --git a/protocols/spicy/bridge.cpp b/protocols/spicy/bridge.cpp new file mode 100644 index 0000000..a39d4e6 --- /dev/null +++ b/protocols/spicy/bridge.cpp @@ -0,0 +1,308 @@ +#include "bridge.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "parsers/http.h" + +static thread_local bool t_thread_ready = false; + +static inline void ensure_thread_ready() { + if (!t_thread_ready) { + hilti::rt::init(); // idempotent, but must run once per thread + spicy::rt::init(); // idempotent, but must run once per thread + t_thread_ready = true; + } +} + +static std::mutex g_mutex; +static bool g_runtime_ready = false; + +static char* strdup_safe(const char* s) { + if (!s) return nullptr; + size_t n = std::strlen(s) + 1; + char* p = static_cast(std::malloc(n)); + if (p) std::memcpy(p, s, n); + return p; +} + +static ParsedField& ensure_slot(ParsedData* dst, const std::string& name) { + if (dst->field_count >= dst->capacity) { + int new_cap = dst->capacity ? dst->capacity * 2 : 16; + dst->fields = static_cast(std::realloc(dst->fields, sizeof(ParsedField) * new_cap)); + dst->capacity = new_cap; + } + ParsedField& f = dst->fields[dst->field_count++]; + f.name = strdup_safe(name.c_str()); + return f; +} + +static void add_field_str(ParsedData* dst, const std::string& name, const std::string& value) { + auto& f = ensure_slot(dst, name); + f.value = strdup_safe(value.c_str()); + f.is_binary = 0; + f.length = static_cast(value.size()); +} + +static void add_field_bin(ParsedData* dst, const std::string& name, const uint8_t* data, size_t len) { + auto& f = ensure_slot(dst, name); + f.value = static_cast(std::malloc(len)); + if (!f.value) { + dst->error_message = strdup_safe("memory allocation failed for binary field"); + return; + } + std::memcpy(f.value, data, len); + f.is_binary = 1; + f.length = static_cast(len); +} + +static std::string scalar_to_string(const hilti::rt::type_info::Value& v) { + const auto& T = v.type(); + + switch (T.tag) { + case hilti::rt::TypeInfo::UnsignedInteger_uint64: + case hilti::rt::TypeInfo::SignedInteger_int64: + case hilti::rt::TypeInfo::UnsignedInteger_uint32: + case hilti::rt::TypeInfo::SignedInteger_int32: + case hilti::rt::TypeInfo::UnsignedInteger_uint16: + case hilti::rt::TypeInfo::SignedInteger_int16: + case hilti::rt::TypeInfo::UnsignedInteger_uint8: + case hilti::rt::TypeInfo::SignedInteger_int8: + case hilti::rt::TypeInfo::Real: + return std::to_string(v); + + case hilti::rt::TypeInfo::Bool: + return T.bool_->get(v) ? "true" : "false"; + + case hilti::rt::TypeInfo::String: + return T.string->get(v); + + case hilti::rt::TypeInfo::Enum: + return std::to_string(v); + + case hilti::rt::TypeInfo::Bytes: { + const auto& b = T.bytes->get(v); + return std::string(reinterpret_cast(b.data()), b.size()); + } + + case hilti::rt::TypeInfo::ValueReference: + return scalar_to_string(T.value_reference->value(v)); + + default: + return ""; + } +} + +static void dump_value(ParsedData* dst, const std::string& prefix, const hilti::rt::type_info::Value& v) { + const auto& T = v.type(); + + if (T.tag == hilti::rt::TypeInfo::ValueReference) { + dump_value(dst, prefix, T.value_reference->value(v)); + return; + } + + if (T.tag == hilti::rt::TypeInfo::Vector) { + size_t idx = 0; + for (const auto& elem : T.vector->iterate(v)) { + std::string key = prefix + "[" + std::to_string(idx++) + "]"; + dump_value(dst, key, elem); + } + return; + } + if (T.tag == hilti::rt::TypeInfo::Set) { + size_t idx = 0; + for (const auto& elem : T.set->iterate(v)) { + std::string key = prefix + "[" + std::to_string(idx++) + "]"; + dump_value(dst, key, elem); + } + return; + } + + if (T.tag == hilti::rt::TypeInfo::Map) { + auto* mt = hilti::rt::type_info::value::auxType(v); + + for (const auto& [k, val] : mt->iterate(v)) { + std::string kstr = scalar_to_string(k); + std::string key = prefix.empty() ? kstr : prefix + "." + kstr; + dump_value(dst, key, val); + } + return; + } + + if (T.tag == hilti::rt::TypeInfo::Optional) { + if (auto inner = T.optional->value(v)) + dump_value(dst, prefix, inner); + else + add_field_str(dst, prefix, ""); + return; + } + + if (T.tag == hilti::rt::TypeInfo::Struct) { + for (const auto& [info, field] : T.struct_->iterate(v)) { + std::string key = prefix.empty() ? info.name : prefix + "." + info.name; + dump_value(dst, key, field); + } + return; + } + + if (T.tag == hilti::rt::TypeInfo::Bytes) { + const auto& b = T.bytes->get(v); + + bool printable = true; + for (auto c : b) { + if (c < 0x20 || c > 0x7e) { printable = false; break; } + } + + if (printable && b.size() <= 256) { + add_field_str(dst, prefix, std::string(reinterpret_cast(b.data()), b.size())); + } + else { + add_field_bin(dst, prefix, reinterpret_cast(b.data()), b.size()); + } + return; + } + + add_field_str(dst, prefix, scalar_to_string(v)); +} + + +void spicy_init() { + std::lock_guard lock(g_mutex); + if (g_runtime_ready) + return; + + hilti::rt::init(); + spicy::rt::init(); + g_runtime_ready = true; +} + +void spicy_cleanup() { + std::lock_guard lock(g_mutex); + if (!g_runtime_ready) + return; + + spicy::rt::done(); + hilti::rt::done(); + g_runtime_ready = false; +} + +int spicy_is_initialized() { + std::lock_guard lock(g_mutex); + return g_runtime_ready ? 1 : 0; +} + +char** spicy_list_parsers(int* count) { + std::lock_guard lock(g_mutex); + + ensure_thread_ready(); + + if (!g_runtime_ready) { + if (count) *count = 0; + return nullptr; + } + + if (!count) return nullptr; + + try { + spicy::rt::Driver drv; + std::stringstream ss; + drv.listParsers(ss); + + std::vector names; + std::string line; + while (std::getline(ss, line)) + if (line.find("::") != std::string::npos) names.emplace_back(line); + + *count = static_cast(names.size()); + if (*count == 0) + return nullptr; + + char** out = static_cast(std::malloc(sizeof(char*) * *count)); + for (int i = 0; i < *count; ++i) + out[i] = strdup_safe(names[i].c_str()); + + return out; + } + catch (const std::exception& e) { + *count = 0; + return nullptr; + } +} + +ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* data, int length) { + if (!parser_name || ! data || length <= 0) + return nullptr; + + auto* res = static_cast(std::calloc(1, sizeof(ParsedData))); + if (!res) + return nullptr; + + res->protocol_name = strdup_safe(parser_name); + + std::lock_guard lock(g_mutex); + + ensure_thread_ready(); + + if (!g_runtime_ready) { + res->error_message = strdup_safe("runtime not initialized"); + return res; + } + + try { + spicy::rt::Driver drv; + auto parser = drv.lookupParser(parser_name); + + if (!parser) { + res->error_message = strdup_safe("parser not found"); + return res; + } + + std::stringstream in; + in.write(reinterpret_cast(data), length); + + auto unit = drv.processInput(**parser, in); + + if (unit && unit->value()) { + dump_value(res, "", unit->value()); + } + else { + res->error_message = strdup_safe("no value returned"); + } + } + catch (const std::exception& e) { + res->error_message = strdup_safe(e.what()); + } + catch (...) { + res->error_message = strdup_safe("unknown C++ exception"); + } + + return res; +} + +void spicy_free_parsed_data(ParsedData* d) { + if (!d) + return; + + for (int i = 0; i < d->field_count; ++i) { + std::free(d->fields[i].name); + std::free(d->fields[i].value); + } + std::free(d->fields); + std::free(d->protocol_name); + std::free(d->error_message); + std::free(d); +} + +void spicy_free_parser_list(char** p, int n) { + if (!p) + return; + + for (int i = 0; i < n; ++i) + std::free(p[i]); + std::free(p); +} \ No newline at end of file diff --git a/protocols/spicy/bridge.h b/protocols/spicy/bridge.h new file mode 100644 index 0000000..e2d6b1f --- /dev/null +++ b/protocols/spicy/bridge.h @@ -0,0 +1,63 @@ +#ifndef GLUTTON_SPICY_BRIDGE_H +#define GLUTTON_SPICY_BRIDGE_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + char* name; // name of field + char* value; // value of field. + int is_binary; // flag (0 or 1) indicating if value is raw bytes + int length; // length of value, essential for binary data +} ParsedField; + +typedef struct { + ParsedField* fields; + int field_count; + int capacity; + char* protocol_name; + char* error_message; +} ParsedData; + +/** + * @brief Initializes the Spicy and HILTI runtimes. + */ +void spicy_init(); + +/** + * @brief Checks if the Spicy runtime is initialized. + * returns 1 if initialized, 0 if not. + */ +int spicy_is_initialized(); + +/** + * @brief Cleans up and shuts down the Spicy and HILTI runtimes. + */ +void spicy_cleanup(); + +/** + * @brief Lists all available (compiled-in) Spicy parsers. + */ +char** spicy_list_parsers(int* count); + +/** + * @brief The main generic parsing function. + */ +ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* data, int length); + +/** + * @brief Frees the memory allocated for a ParsedData struct. + */ +void spicy_free_parsed_data(ParsedData* data); + +/** + * @brief Frees the memory allocated for the parser list returned by spicy_list_parsers. + */ +void spicy_free_parser_list(char** parsers, int count); + +#ifdef __cplusplus +} +#endif + +#endif // GLUTTON_SPICY_BRIDGE_H \ No newline at end of file diff --git a/protocols/spicy/handlers/http.go b/protocols/spicy/handlers/http.go new file mode 100644 index 0000000..0715522 --- /dev/null +++ b/protocols/spicy/handlers/http.go @@ -0,0 +1,203 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "net" + "strconv" + "strings" + + "github.com/mushorg/glutton/connection" + "github.com/mushorg/glutton/producer" + "github.com/mushorg/glutton/protocols/interfaces" + "github.com/mushorg/glutton/protocols/spicy" + "github.com/mushorg/glutton/protocols/tcp" +) + +func sendJSON(conn net.Conn, b []byte) error { + _, err := conn.Write( + append([]byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(b))), b...), + ) + return err +} + +func writePlainOK(conn net.Conn) error { + _, err := conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n")) + return err +} + +func handleEthereumRPC(body []byte, conn net.Conn) bool { + if !bytes.Contains(body, []byte("eth_blockNumber")) { + return false + } + resp := struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Result string `json:"result"` + }{ + JSONRPC: "2.0", + ID: 0, + Result: "0x2ecd9e", + } + b, _ := json.Marshal(resp) + _ = sendJSON(conn, b) + return true +} + +func handleYarnNewApplication(method, uri string, conn net.Conn) bool { + if method != "POST" || !strings.Contains(uri, "cluster/apps/new-application") { + return false + } + resp, _ := json.Marshal(&struct { + ApplicationID string `json:"application-id"` + MaximumResourceCapability interface{} `json:"maximum-resource-capability"` + }{ + ApplicationID: "application_1527144634877_20465", + MaximumResourceCapability: struct { + Memory int `json:"memory"` + VCores int `json:"vCores"` + }{Memory: 16384, VCores: 8}, + }) + _ = sendJSON(conn, resp) + return true +} + +func handleWallet(uri string, conn net.Conn) bool { + if !strings.Contains(uri, "wallet") { + return false + } + body := []byte(`[[""]]`) + header := fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(body)) + conn.Write([]byte(header)) + conn.Write(body) + return true +} + +func handleDockerAPIVersion(uri string, conn net.Conn) bool { + if !strings.HasPrefix(uri, "/v1.16/version") { + return false + } + if data, err := tcp.Res.ReadFile("resources/docker_api.json"); err == nil { + conn.Write(append([]byte(fmt.Sprintf( + "HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(data))), data...)) + } + return true +} + +func handleCitrixSMB(uri string, conn net.Conn) bool { + if !strings.HasPrefix(uri, "/vpn/") { + return false + } + headers := `Server: Apache +X-Frame-Options: SAMEORIGIN +Last-Modified: Thu, 28 Nov 2019 20:19:22 GMT +ETag: "53-5986dd42b0680" +Accept-Ranges: bytes +Content-Length: 93 +X-XSS-Protection: 1; mode=block +X-Content-Type-Options: nosniff +Content-Type: text/plain; charset=UTF-8` + smbCfg := "\r\n\r\n[global]\r\n\tencrypt passwords = yes\r\n\tname resolve order = lmhosts wins host bcast\r\n" + conn.Write([]byte("HTTP/1.1 200 OK\r\n" + headers + smbCfg)) + return true +} + +func handleVMwareSend(ctx context.Context, body []byte, uri string, md connection.Metadata, log interfaces.Logger, hp interfaces.Honeypot) bool { + if !strings.Contains(uri, "hyper/send") || len(body) == 0 { + return false + } + parts := strings.Split(string(body), " ") + if len(parts) < 11 { + return false + } + c, err := net.Dial("tcp", parts[9]+":"+parts[10]) + if err != nil { + log.Error("vmware-send dial failed", producer.ErrAttr(err)) + return true + } + go func() { + if err := tcp.HandleTCP(ctx, c, md, log, hp); err != nil { + log.Error("vmware-send TCP relay error", producer.ErrAttr(err)) + } + }() + return true +} + +func HandleHTTP(ctx context.Context, conn net.Conn, md connection.Metadata, log interfaces.Logger, hp interfaces.Honeypot) error { + + defer conn.Close() + + payload, err := spicy.ReadInitialBytes("http", conn) + if err != nil { + return err + } + if len(payload) == 0 { + return nil + } + + parsed, err := spicy.Parse("http", payload) + if err != nil { + log.Error("spicy parse error", producer.ErrAttr(err)) + _ = hp.ProduceTCP("spicy-http-failed", conn, md, payload, + map[string]string{"error": err.Error()}) + return err + } + + method, _ := parsed.Fields["method"].(string) + uri, _ := parsed.Fields["uri"].(string) + version, _ := parsed.Fields["version.number"].(string) + + method = strings.ToUpper(method) + + var body []byte + if v, ok := parsed.Fields["body.content"]; ok { + switch b := v.(type) { + case []byte: + body = b + case string: + body = []byte(b) + } + } + + path, query := uri, "" + if sp := strings.SplitN(uri, "?", 2); len(sp) == 2 { + path, query = sp[0], sp[1] + } + + host, port, _ := net.SplitHostPort(conn.RemoteAddr().String()) + log.Info(fmt.Sprintf("HTTP %s %s request handled: %s", version, method, path), // added "version" as a proof of concept, not identical to the original pure Go parser + slog.String("handler", "spicy-http"), + slog.String("dest_port", strconv.Itoa(int(md.TargetPort))), + slog.String("src_ip", host), + slog.String("src_port", port), + slog.String("path", path), + slog.String("query", query), + ) + + if len(body) > 0 { + max := len(body) + if max > 1024 { + max = 1024 + } + log.Info("HTTP payload:\n" + hex.Dump(body[:max])) + } + + _ = hp.ProduceTCP("http", conn, md, payload, parsed) + + handled := false + switch method { + case "POST": + handled = handleEthereumRPC(body, conn) || handleYarnNewApplication(method, uri, conn) + } + + handled = handled || handleWallet(uri, conn) || handleDockerAPIVersion(uri, conn) || handleCitrixSMB(uri, conn) || handleVMwareSend(ctx, body, uri, md, log, hp) + + if !handled { + _ = writePlainOK(conn) + } + return nil +} diff --git a/protocols/spicy/helpers.go b/protocols/spicy/helpers.go new file mode 100644 index 0000000..7e36d03 --- /dev/null +++ b/protocols/spicy/helpers.go @@ -0,0 +1,135 @@ +package spicy + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "io" + "net" + "regexp" + "strconv" + "strings" +) + +func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { + switch protocol { + + case "http": + r := bufio.NewReader(conn) + raw := make([]byte, 0, 4096) + + for { + line, err := r.ReadBytes('\n') + if err != nil { + return nil, err + } + raw = append(raw, line...) + if bytes.Equal(line, []byte("\r\n")) { + break + } + } + + var clen int + if m := regexp.MustCompile(`(?i)content-length:\s*(\d+)`).FindSubmatch(raw); m != nil { + clen, _ = strconv.Atoi(string(m[1])) + } + + if clen > 0 { + body := make([]byte, clen) + if _, err := io.ReadFull(r, body); err != nil { + return nil, err + } + raw = append(raw, body...) + } + return raw, nil + + case "dns": + var lenBuf [2]byte + if _, err := io.ReadFull(conn, lenBuf[:]); err != nil { + return nil, err + } + l := int(binary.BigEndian.Uint16(lenBuf[:])) + if l == 0 || l > 64*1024 { + return nil, errors.New("suspicious DNS length") + } + p := make([]byte, l) + _, err := io.ReadFull(conn, p) + return p, err + + default: + buf := make([]byte, 8192) + n, err := conn.Read(buf) + return buf[:n], err + } +} + +func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { + root := map[string]interface{}{} + + for k, v := range flat { + cur := root + parts := strings.Split(k, ".") + + for i, p := range parts { + if b := strings.Index(p, "["); b != -1 { + base := p[:b] + e := strings.Index(p[b:], "]") + idx, _ := strconv.Atoi(p[b+1 : b+e]) + + slice, ok := cur[base].([]interface{}) + if !ok { + slice = make([]interface{}, idx+1) + cur[base] = slice + } else if idx >= len(slice) { + slice = append(slice, make([]interface{}, idx+1-len(slice))...) + cur[base] = slice + } + + if i == len(parts)-1 { + slice[idx] = v + } else { + if slice[idx] == nil { + slice[idx] = map[string]interface{}{} + } + cur = slice[idx].(map[string]interface{}) + } + } else { + if i == len(parts)-1 { + cur[p] = v + } else { + if _, ok := cur[p]; !ok { + cur[p] = map[string]interface{}{} + } + cur = cur[p].(map[string]interface{}) + } + } + } + } + return root +} + +func GetDeepStr(m map[string]interface{}, path ...string) string { + for _, p := range path { + parts := strings.Split(p, ".") + cur := m + for i, seg := range parts { + v, ok := cur[seg] + if !ok { + break + } + if i == len(parts)-1 { + if s, ok := v.(string); ok { + return s + } + } else { + if nxt, ok := v.(map[string]interface{}); ok { + cur = nxt + } else { + break + } + } + } + } + return "" +} diff --git a/protocols/spicy/parser.go b/protocols/spicy/parser.go new file mode 100644 index 0000000..f5f6052 --- /dev/null +++ b/protocols/spicy/parser.go @@ -0,0 +1,195 @@ +package spicy + +/* +#cgo CXXFLAGS: -I/opt/spicy/include -I${SRCDIR}/parsers -std=c++17 -fPIC -O3 -DNDEBUG -fvisibility=hidden +#cgo LDFLAGS: -L/opt/spicy/lib -lspicy-rt -lhilti-rt -lz -lpthread -ldl "-Wl,-rpath,/opt/spicy/lib" +#include +#include "bridge.h" +*/ +import "C" + +import ( + "errors" + "fmt" + "runtime" + "strings" + "sync" + "unsafe" + + "github.com/mushorg/glutton/protocols/interfaces" +) + +type workerCmdKind int8 + +const ( + cmdInitAndList workerCmdKind = iota + 1 + cmdParse + cmdShutdown +) + +type workerCmd struct { + kind workerCmdKind + parser string + data []byte + replyChan chan any +} + +var ( + workerOnce sync.Once + cmdCh chan workerCmd +) + +func startWorker() { + workerOnce.Do(func() { + cmdCh = make(chan workerCmd) + go func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + for cmd := range cmdCh { + switch cmd.kind { + case cmdInitAndList: + C.spicy_init() + + cnt := C.int(0) + pp := C.spicy_list_parsers(&cnt) + list := []string{} + if pp != nil && cnt > 0 { + ptrs := (*[1 << 30]*C.char)(unsafe.Pointer(pp))[:cnt:cnt] + for _, p := range ptrs { + list = append(list, C.GoString(p)) + } + C.spicy_free_parser_list(pp, cnt) + } + cmd.replyChan <- list + + case cmdParse: + cn := C.CString(cmd.parser) + cres := C.spicy_parse_generic( + cn, + (*C.uchar)(unsafe.Pointer(&cmd.data[0])), + C.int(len(cmd.data)), + ) + C.free(unsafe.Pointer(cn)) + cmd.replyChan <- cres + + case cmdShutdown: + var err error + func() { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic during cleanup: %v", r) + } + }() + C.spicy_cleanup() + }() + cmd.replyChan <- err + close(cmd.replyChan) + return + } + } + }() + }) +} + +var ( + initOnce sync.Once + registeredParsers = make(map[string]string) + parsersMutex sync.RWMutex +) + +func Initialize(logger interfaces.Logger) { + initOnce.Do(func() { + startWorker() + + resp := make(chan any, 1) + cmdCh <- workerCmd{kind: cmdInitAndList, replyChan: resp} + names := (<-resp).([]string) + + if C.spicy_is_initialized() == 0 { + logger.Error("failed to initialise Spicy runtime") + return + } + logger.Info("Spicy runtime initialised successfully") + + parsersMutex.Lock() + for _, raw := range names { + raw = strings.TrimSpace(raw) + + if raw == "" || !strings.Contains(raw, "::") { + continue + } + + parts := strings.SplitN(raw, "::", 2) + proto := strings.ToLower(strings.TrimSpace(parts[0])) + canonical := strings.TrimSpace(raw) + + if _, ok := registeredParsers[proto]; !ok { + registeredParsers[proto] = canonical + logger.Info("registered Spicy parser", "protocol", proto, "parser", canonical) + } + } + parsersMutex.Unlock() + }) +} + +type ParsedData struct { + Protocol string `json:"protocol"` + Fields map[string]interface{} `json:"fields"` + Error error `json:"-"` +} + +func Parse(proto string, data []byte) (*ParsedData, error) { + parsersMutex.RLock() + name, ok := registeredParsers[strings.ToLower(proto)] + parsersMutex.RUnlock() + if !ok { + return nil, fmt.Errorf("no Spicy parser registered for %q", proto) + } + name = strings.TrimSpace(name) + + if len(data) == 0 { + return nil, errors.New("input data is empty") + } + + resp := make(chan any, 1) + cmdCh <- workerCmd{kind: cmdParse, parser: name, data: data, replyChan: resp} + raw := <-resp + if raw == nil { + return nil, errors.New("Spicy parse failed: no response received") + } + cRes, ok := raw.(*C.ParsedData) + if !ok { + return nil, errors.New("internal type assertion failed") + } + defer C.spicy_free_parsed_data(cRes) + + out := &ParsedData{Protocol: proto, Fields: map[string]interface{}{}} + if cRes.error_message != nil { + err := errors.New(C.GoString(cRes.error_message)) + out.Error = err + return out, err + } + + if cRes.fields != nil && cRes.field_count > 0 { + fs := (*[1 << 30]C.ParsedField)(unsafe.Pointer(cRes.fields))[:cRes.field_count:cRes.field_count] + for _, f := range fs { + k := C.GoString(f.name) + if f.is_binary != 0 { + out.Fields[k] = C.GoBytes(unsafe.Pointer(f.value), f.length) + } else { + out.Fields[k] = C.GoString(f.value) + } + } + } + return out, nil +} + +func Cleanup() error { + resp := make(chan any, 1) + cmdCh <- workerCmd{kind: cmdShutdown, replyChan: resp} + if v := <-resp; v != nil { + return v.(error) + } + return nil +} diff --git a/protocols/spicy/parsers/http.spicy b/protocols/spicy/parsers/http.spicy new file mode 100644 index 0000000..ff5103d --- /dev/null +++ b/protocols/spicy/parsers/http.spicy @@ -0,0 +1,37 @@ +module HTTP; + +const Token = /[^ \t\r\n:]+/; +const HeaderName = /[^:\r\n]+/; +const WhiteSpace = /[ \t]+/; +const OptionalWhiteSpace = /[ \t]*/; +const NewLine = /\r?\n/; +const RestOfLine = /[^\r\n]*/; + +type Version = unit { + : /HTTP\//; + number: /[0-9]+\.[0-9]*/; +}; + +type Header = unit { + name: HeaderName; + : /:/; + : OptionalWhiteSpace; + value: RestOfLine; + : NewLine; +}; + +type Body = unit { + content: bytes &eod; +}; + +public type Request = unit { + method: Token; + : WhiteSpace; + uri: Token; + : WhiteSpace; + version: Version; + : NewLine; + headers: Header[]; + : NewLine; + body: Body; +}; \ No newline at end of file diff --git a/protocols/tcp/http.go b/protocols/tcp/http.go index 221ffbb..6289330 100644 --- a/protocols/tcp/http.go +++ b/protocols/tcp/http.go @@ -180,7 +180,7 @@ func HandleHTTP(ctx context.Context, conn net.Conn, md connection.Metadata, logg } if strings.Contains(req.RequestURI, "/v1.16/version") { - data, err := res.ReadFile("resources/docker_api.json") + data, err := Res.ReadFile("resources/docker_api.json") if err != nil { return fmt.Errorf("failed to read embedded file: %w", err) } diff --git a/protocols/tcp/resources.go b/protocols/tcp/resources.go index 6c68fac..dd922f3 100644 --- a/protocols/tcp/resources.go +++ b/protocols/tcp/resources.go @@ -4,5 +4,5 @@ import "embed" var ( //go:embed resources - res embed.FS + Res embed.FS ) From d9a6e3b2ae7d8843b26b1fc7b7897e11690141f8 Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Sun, 6 Jul 2025 21:48:14 +0530 Subject: [PATCH 02/13] feat: adds basic and relevant function documentation for bridge and parser --- protocols/spicy/bridge.cpp | 27 +++++++++++++++++++++++---- protocols/spicy/handlers/http.go | 6 ++++-- protocols/spicy/helpers.go | 5 +++++ protocols/spicy/parser.go | 18 ++++++++++++++---- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/protocols/spicy/bridge.cpp b/protocols/spicy/bridge.cpp index a39d4e6..3a7f75e 100644 --- a/protocols/spicy/bridge.cpp +++ b/protocols/spicy/bridge.cpp @@ -10,8 +10,14 @@ #include "parsers/http.h" +// tracks runtime init per thread static thread_local bool t_thread_ready = false; +// global mutex for runtime init sync +static std::mutex g_mutex; +static bool g_runtime_ready = false; + +// initializes HILTI and Spicy for current thread static inline void ensure_thread_ready() { if (!t_thread_ready) { hilti::rt::init(); // idempotent, but must run once per thread @@ -20,9 +26,7 @@ static inline void ensure_thread_ready() { } } -static std::mutex g_mutex; -static bool g_runtime_ready = false; - +// safely duplicates C string, handling NULL input static char* strdup_safe(const char* s) { if (!s) return nullptr; size_t n = std::strlen(s) + 1; @@ -31,6 +35,8 @@ static char* strdup_safe(const char* s) { return p; } +// ensures space in the ParsedData fields array and returns +// reference to the next available field slot static ParsedField& ensure_slot(ParsedData* dst, const std::string& name) { if (dst->field_count >= dst->capacity) { int new_cap = dst->capacity ? dst->capacity * 2 : 16; @@ -42,6 +48,7 @@ static ParsedField& ensure_slot(ParsedData* dst, const std::string& name) { return f; } +// adds a string field to the ParsedData structure static void add_field_str(ParsedData* dst, const std::string& name, const std::string& value) { auto& f = ensure_slot(dst, name); f.value = strdup_safe(value.c_str()); @@ -49,6 +56,7 @@ static void add_field_str(ParsedData* dst, const std::string& name, const std::s f.length = static_cast(value.size()); } +// adds a binary field to the ParsedData structure static void add_field_bin(ParsedData* dst, const std::string& name, const uint8_t* data, size_t len) { auto& f = ensure_slot(dst, name); f.value = static_cast(std::malloc(len)); @@ -61,6 +69,8 @@ static void add_field_bin(ParsedData* dst, const std::string& name, const uint8_ f.length = static_cast(len); } +// converts HILTI scalar value to its string representation and handles +// various HILTI type system values properly static std::string scalar_to_string(const hilti::rt::type_info::Value& v) { const auto& T = v.type(); @@ -98,6 +108,9 @@ static std::string scalar_to_string(const hilti::rt::type_info::Value& v) { } } +// recursively extracts and stores all fields from a HILTI value, +// creates flat field names using dot notation for +// nested structures and bracket notation for array indices static void dump_value(ParsedData* dst, const std::string& prefix, const hilti::rt::type_info::Value& v) { const auto& T = v.type(); @@ -170,7 +183,7 @@ static void dump_value(ParsedData* dst, const std::string& prefix, const hilti:: add_field_str(dst, prefix, scalar_to_string(v)); } - +// initializes HILTI and Spicy runtimes globally void spicy_init() { std::lock_guard lock(g_mutex); if (g_runtime_ready) @@ -181,6 +194,7 @@ void spicy_init() { g_runtime_ready = true; } +// cleans up HILTI and Spicy runtimes globally void spicy_cleanup() { std::lock_guard lock(g_mutex); if (!g_runtime_ready) @@ -191,11 +205,13 @@ void spicy_cleanup() { g_runtime_ready = false; } +// checks if the Spicy runtime is initialized int spicy_is_initialized() { std::lock_guard lock(g_mutex); return g_runtime_ready ? 1 : 0; } +// lists all available Spicy parsers and returns their names char** spicy_list_parsers(int* count) { std::lock_guard lock(g_mutex); @@ -234,6 +250,7 @@ char** spicy_list_parsers(int* count) { } } +// parses data using a specified Spicy parser and returns the parsed data ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* data, int length) { if (!parser_name || ! data || length <= 0) return nullptr; @@ -284,6 +301,7 @@ ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* da return res; } +// frees the memory allocated for ParsedData and its fields void spicy_free_parsed_data(ParsedData* d) { if (!d) return; @@ -298,6 +316,7 @@ void spicy_free_parsed_data(ParsedData* d) { std::free(d); } +// frees the memory allocated for a list of parser names void spicy_free_parser_list(char** p, int n) { if (!p) return; diff --git a/protocols/spicy/handlers/http.go b/protocols/spicy/handlers/http.go index 0715522..47a0f3f 100644 --- a/protocols/spicy/handlers/http.go +++ b/protocols/spicy/handlers/http.go @@ -18,6 +18,9 @@ import ( "github.com/mushorg/glutton/protocols/tcp" ) +// Identical implementation of the original Go HTTP handler, but using Spicy for parsing +// I've tried to keep the logs and responses as close to the original as possible + func sendJSON(conn net.Conn, b []byte) error { _, err := conn.Write( append([]byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(b))), b...), @@ -128,7 +131,6 @@ func handleVMwareSend(ctx context.Context, body []byte, uri string, md connectio } func HandleHTTP(ctx context.Context, conn net.Conn, md connection.Metadata, log interfaces.Logger, hp interfaces.Honeypot) error { - defer conn.Close() payload, err := spicy.ReadInitialBytes("http", conn) @@ -139,7 +141,7 @@ func HandleHTTP(ctx context.Context, conn net.Conn, md connection.Metadata, log return nil } - parsed, err := spicy.Parse("http", payload) + parsed, err := spicy.Parse("http", payload) // parse the HTTP request using Spicy if err != nil { log.Error("spicy parse error", producer.ErrAttr(err)) _ = hp.ProduceTCP("spicy-http-failed", conn, md, payload, diff --git a/protocols/spicy/helpers.go b/protocols/spicy/helpers.go index 7e36d03..3816aec 100644 --- a/protocols/spicy/helpers.go +++ b/protocols/spicy/helpers.go @@ -12,6 +12,8 @@ import ( "strings" ) +// reads protocol-specific initial data from a network connection and +// returns the complete protocol message as a byte slice. func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { switch protocol { @@ -64,6 +66,8 @@ func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { } } +// converts a flat map with dot notation keys into a nested map structure. +// created initially as a Spicy helper to handle nested data structures func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { root := map[string]interface{}{} @@ -109,6 +113,7 @@ func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { return root } +// retrieves a string value from a nested map using a path with dot notation func GetDeepStr(m map[string]interface{}, path ...string) string { for _, p := range path { parts := strings.Split(p, ".") diff --git a/protocols/spicy/parser.go b/protocols/spicy/parser.go index f5f6052..e05d1bf 100644 --- a/protocols/spicy/parser.go +++ b/protocols/spicy/parser.go @@ -39,11 +39,12 @@ var ( cmdCh chan workerCmd ) +// initializes the Spicy worker thread if not already started func startWorker() { workerOnce.Do(func() { cmdCh = make(chan workerCmd) go func() { - runtime.LockOSThread() + runtime.LockOSThread() // lock to OS thread for C++ runtime thread-local storage defer runtime.UnlockOSThread() for cmd := range cmdCh { @@ -93,9 +94,9 @@ func startWorker() { } var ( - initOnce sync.Once + initOnce sync.Once // ensures Spicy runtime is initialized only once registeredParsers = make(map[string]string) - parsersMutex sync.RWMutex + parsersMutex sync.RWMutex // protects access to registeredParsers ) func Initialize(logger interfaces.Logger) { @@ -120,6 +121,7 @@ func Initialize(logger interfaces.Logger) { continue } + // protocol names look like "HTTP::Request", so we split on "::" parts := strings.SplitN(raw, "::", 2) proto := strings.ToLower(strings.TrimSpace(parts[0])) canonical := strings.TrimSpace(raw) @@ -133,15 +135,18 @@ func Initialize(logger interfaces.Logger) { }) } +// represents the result of parsing protocol data with Spicy type ParsedData struct { Protocol string `json:"protocol"` Fields map[string]interface{} `json:"fields"` Error error `json:"-"` } +// analyzes protocol data using the appropriate Spicy parser +// the parser is automatically selected based on the protocol name func Parse(proto string, data []byte) (*ParsedData, error) { parsersMutex.RLock() - name, ok := registeredParsers[strings.ToLower(proto)] + name, ok := registeredParsers[strings.ToLower(proto)] // parser lookup parsersMutex.RUnlock() if !ok { return nil, fmt.Errorf("no Spicy parser registered for %q", proto) @@ -171,13 +176,16 @@ func Parse(proto string, data []byte) (*ParsedData, error) { return out, err } + // extract parsed fields if cRes.fields != nil && cRes.field_count > 0 { fs := (*[1 << 30]C.ParsedField)(unsafe.Pointer(cRes.fields))[:cRes.field_count:cRes.field_count] for _, f := range fs { k := C.GoString(f.name) if f.is_binary != 0 { + // binary out.Fields[k] = C.GoBytes(unsafe.Pointer(f.value), f.length) } else { + // string out.Fields[k] = C.GoString(f.value) } } @@ -185,6 +193,8 @@ func Parse(proto string, data []byte) (*ParsedData, error) { return out, nil } +// shuts down the Spicy runtime and releases all associated resources +// it's probably safe to call multiple times func Cleanup() error { resp := make(chan any, 1) cmdCh <- workerCmd{kind: cmdShutdown, replyChan: resp} From 722a49e027c4a339c860920e29876f6b10bfe5ab Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Mon, 7 Jul 2025 03:42:49 +0530 Subject: [PATCH 03/13] chore: hardens Spicy bridge - OOM-safe growth, parse timeout, HTTP size cap & init errors --- glutton.go | 4 +++- protocols/spicy/bridge.cpp | 36 ++++++++++++++++++++++++++---------- protocols/spicy/helpers.go | 23 ++++++++++++++++++++--- protocols/spicy/parser.go | 21 ++++++++++++++++++--- 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/glutton.go b/glutton.go index fb5a62c..c47ffc6 100644 --- a/glutton.go +++ b/glutton.go @@ -137,7 +137,9 @@ func (g *Glutton) Init() error { // Initializing Spicy parsers if viper.GetBool("spicy.enabled") { - spicy.Initialize(g.Logger) + if err := spicy.Initialize(g.Logger); err != nil { + return fmt.Errorf("failed to initialize Spicy: %w", err) + } } return nil diff --git a/protocols/spicy/bridge.cpp b/protocols/spicy/bridge.cpp index 3a7f75e..048a2b7 100644 --- a/protocols/spicy/bridge.cpp +++ b/protocols/spicy/bridge.cpp @@ -38,11 +38,20 @@ static char* strdup_safe(const char* s) { // ensures space in the ParsedData fields array and returns // reference to the next available field slot static ParsedField& ensure_slot(ParsedData* dst, const std::string& name) { - if (dst->field_count >= dst->capacity) { - int new_cap = dst->capacity ? dst->capacity * 2 : 16; - dst->fields = static_cast(std::realloc(dst->fields, sizeof(ParsedField) * new_cap)); + if ( dst->field_count >= dst->capacity ) { // need to grow the array + const int new_cap = dst->capacity ? dst->capacity * 2 : 16; + + // realloc safety check + void* mem = std::realloc(dst->fields, sizeof(ParsedField) * new_cap); + if (!mem) { // memory allocation failed + static ParsedField dummy; // placeholder + dst->error_message = strdup_safe("out of memory while growing ParsedField slot array"); + return dummy; + } + dst->fields = static_cast(mem); dst->capacity = new_cap; } + ParsedField& f = dst->fields[dst->field_count++]; f.name = strdup_safe(name.c_str()); return f; @@ -108,10 +117,17 @@ static std::string scalar_to_string(const hilti::rt::type_info::Value& v) { } } +static constexpr int kMaxDepth = 64; // maximum recursion depth for nested structures + // recursively extracts and stores all fields from a HILTI value, // creates flat field names using dot notation for // nested structures and bracket notation for array indices -static void dump_value(ParsedData* dst, const std::string& prefix, const hilti::rt::type_info::Value& v) { +static void dump_value(ParsedData* dst, const std::string& prefix, const hilti::rt::type_info::Value& v, int depth = 0) { + if (depth > kMaxDepth) { + add_field_str(dst, prefix, ""); + return; + } + const auto& T = v.type(); if (T.tag == hilti::rt::TypeInfo::ValueReference) { @@ -123,7 +139,7 @@ static void dump_value(ParsedData* dst, const std::string& prefix, const hilti:: size_t idx = 0; for (const auto& elem : T.vector->iterate(v)) { std::string key = prefix + "[" + std::to_string(idx++) + "]"; - dump_value(dst, key, elem); + dump_value(dst, key, elem, depth + 1); } return; } @@ -131,7 +147,7 @@ static void dump_value(ParsedData* dst, const std::string& prefix, const hilti:: size_t idx = 0; for (const auto& elem : T.set->iterate(v)) { std::string key = prefix + "[" + std::to_string(idx++) + "]"; - dump_value(dst, key, elem); + dump_value(dst, key, elem, depth + 1); } return; } @@ -142,14 +158,14 @@ static void dump_value(ParsedData* dst, const std::string& prefix, const hilti:: for (const auto& [k, val] : mt->iterate(v)) { std::string kstr = scalar_to_string(k); std::string key = prefix.empty() ? kstr : prefix + "." + kstr; - dump_value(dst, key, val); + dump_value(dst, key, val, depth + 1); } return; } if (T.tag == hilti::rt::TypeInfo::Optional) { if (auto inner = T.optional->value(v)) - dump_value(dst, prefix, inner); + dump_value(dst, prefix, inner, depth + 1); else add_field_str(dst, prefix, ""); return; @@ -158,7 +174,7 @@ static void dump_value(ParsedData* dst, const std::string& prefix, const hilti:: if (T.tag == hilti::rt::TypeInfo::Struct) { for (const auto& [info, field] : T.struct_->iterate(v)) { std::string key = prefix.empty() ? info.name : prefix + "." + info.name; - dump_value(dst, key, field); + dump_value(dst, key, field, depth + 1); } return; } @@ -285,7 +301,7 @@ ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* da auto unit = drv.processInput(**parser, in); if (unit && unit->value()) { - dump_value(res, "", unit->value()); + dump_value(res, "", unit->value(), 0); } else { res->error_message = strdup_safe("no value returned"); diff --git a/protocols/spicy/helpers.go b/protocols/spicy/helpers.go index 3816aec..307d988 100644 --- a/protocols/spicy/helpers.go +++ b/protocols/spicy/helpers.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/binary" "errors" + "fmt" "io" "net" "regexp" @@ -12,12 +13,17 @@ import ( "strings" ) +// package level regex to match conlen header in HTTP requests +var contentLenRE = regexp.MustCompile(`(?i)Content-Length:\s*(\d+)`) + // reads protocol-specific initial data from a network connection and // returns the complete protocol message as a byte slice. func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { switch protocol { case "http": + const maxHTTPBody = 1 << 20 // 1 MiB limit + r := bufio.NewReader(conn) raw := make([]byte, 0, 4096) @@ -33,10 +39,14 @@ func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { } var clen int - if m := regexp.MustCompile(`(?i)content-length:\s*(\d+)`).FindSubmatch(raw); m != nil { + if m := contentLenRE.FindSubmatch(raw); m != nil { clen, _ = strconv.Atoi(string(m[1])) } + if clen > maxHTTPBody { + return nil, fmt.Errorf("Content-Length %d exceeds maximum %d", clen, maxHTTPBody) + } + if clen > 0 { body := make([]byte, clen) if _, err := io.ReadFull(r, body); err != nil { @@ -96,7 +106,10 @@ func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { if slice[idx] == nil { slice[idx] = map[string]interface{}{} } - cur = slice[idx].(map[string]interface{}) + cur, ok = slice[idx].(map[string]interface{}) + if !ok { + return nil + } } } else { if i == len(parts)-1 { @@ -105,7 +118,11 @@ func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { if _, ok := cur[p]; !ok { cur[p] = map[string]interface{}{} } - cur = cur[p].(map[string]interface{}) + m, ok := cur[p].(map[string]interface{}) + if !ok { + return nil + } + cur = m } } } diff --git a/protocols/spicy/parser.go b/protocols/spicy/parser.go index e05d1bf..363afc0 100644 --- a/protocols/spicy/parser.go +++ b/protocols/spicy/parser.go @@ -14,6 +14,7 @@ import ( "runtime" "strings" "sync" + "time" "unsafe" "github.com/mushorg/glutton/protocols/interfaces" @@ -99,7 +100,9 @@ var ( parsersMutex sync.RWMutex // protects access to registeredParsers ) -func Initialize(logger interfaces.Logger) { +var initErr error + +func Initialize(logger interfaces.Logger) error { initOnce.Do(func() { startWorker() @@ -108,7 +111,8 @@ func Initialize(logger interfaces.Logger) { names := (<-resp).([]string) if C.spicy_is_initialized() == 0 { - logger.Error("failed to initialise Spicy runtime") + initErr = errors.New("failed to initialise Spicy runtime") + logger.Error(initErr.Error()) return } logger.Info("Spicy runtime initialised successfully") @@ -133,6 +137,7 @@ func Initialize(logger interfaces.Logger) { } parsersMutex.Unlock() }) + return initErr } // represents the result of parsing protocol data with Spicy @@ -142,6 +147,8 @@ type ParsedData struct { Error error `json:"-"` } +const parseTimeout = 10 * time.Second + // analyzes protocol data using the appropriate Spicy parser // the parser is automatically selected based on the protocol name func Parse(proto string, data []byte) (*ParsedData, error) { @@ -159,10 +166,18 @@ func Parse(proto string, data []byte) (*ParsedData, error) { resp := make(chan any, 1) cmdCh <- workerCmd{kind: cmdParse, parser: name, data: data, replyChan: resp} - raw := <-resp + + var raw any + select { + case raw = <-resp: // normal path + case <-time.After(parseTimeout): // worker stalled + return nil, fmt.Errorf("Spicy parse timed-out after %s", parseTimeout) + } + if raw == nil { return nil, errors.New("Spicy parse failed: no response received") } + cRes, ok := raw.(*C.ParsedData) if !ok { return nil, errors.New("internal type assertion failed") From c7c1e39ab8382630beb344b33bc4e4435557e9a8 Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Mon, 7 Jul 2025 04:19:45 +0530 Subject: [PATCH 04/13] chore: hardens C++ glue, adds depth guard & OOM-safe field handling * reserves field slots without duplicating names; avoid double-free / leak * detects and aborts on strdup / malloc failures, rolling back field_count * adds early-exit helper * adds clarifying comments --- protocols/spicy/bridge.cpp | 75 ++++++++++++++++++++++++++++++++------ protocols/spicy/helpers.go | 2 +- protocols/spicy/parser.go | 10 ++++- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/protocols/spicy/bridge.cpp b/protocols/spicy/bridge.cpp index 048a2b7..cac91f4 100644 --- a/protocols/spicy/bridge.cpp +++ b/protocols/spicy/bridge.cpp @@ -17,6 +17,9 @@ static thread_local bool t_thread_ready = false; static std::mutex g_mutex; static bool g_runtime_ready = false; +// Note: Spicy/HILTI keep thread-local state, so we must call init() +// once on every OS thread that touches the runtime. The worker +// goroutine is pinned with runtime.LockOSThread() on the Go side // initializes HILTI and Spicy for current thread static inline void ensure_thread_ready() { if (!t_thread_ready) { @@ -26,7 +29,15 @@ static inline void ensure_thread_ready() { } } +// helper for early bail out if a previous allocation failed +// add_field_*() and dump_value() check this so that, after the first +// OOM, we stop allocating and just walk the data +static inline bool has_error(const ParsedData* d) { + return d && d->error_message; +} + // safely duplicates C string, handling NULL input +// return NULL on OOM; callers must check static char* strdup_safe(const char* s) { if (!s) return nullptr; size_t n = std::strlen(s) + 1; @@ -37,8 +48,10 @@ static char* strdup_safe(const char* s) { // ensures space in the ParsedData fields array and returns // reference to the next available field slot -static ParsedField& ensure_slot(ParsedData* dst, const std::string& name) { - if ( dst->field_count >= dst->capacity ) { // need to grow the array +// if realloc failed, sets error_message and returns a static dummy slot +// safe because of single worker thread model +static ParsedField& ensure_slot(ParsedData* dst) { + if (dst->field_count >= dst->capacity) { // need to grow the array const int new_cap = dst->capacity ? dst->capacity * 2 : 16; // realloc safety check @@ -52,25 +65,64 @@ static ParsedField& ensure_slot(ParsedData* dst, const std::string& name) { dst->capacity = new_cap; } - ParsedField& f = dst->fields[dst->field_count++]; - f.name = strdup_safe(name.c_str()); + ParsedField& f = dst->fields[dst->field_count++]; + f.name = nullptr; + f.value = nullptr; + f.is_binary = 0; + f.length = 0; return f; } // adds a string field to the ParsedData structure -static void add_field_str(ParsedData* dst, const std::string& name, const std::string& value) { - auto& f = ensure_slot(dst, name); +static void add_field_str(ParsedData* dst, const std::string& name, const std::string& value){ + if (has_error(dst)) return; + + auto& f = ensure_slot(dst); + if (has_error(dst)) return; + + f.name = strdup_safe(name.c_str()); + if (!f.name) { + dst->error_message = strdup_safe("out of memory duplicating field name"); + --dst->field_count; // revert field count increment + return; + } f.value = strdup_safe(value.c_str()); + if (!f.value) { + dst->error_message = strdup_safe("out of memory duplicating field value"); + std::free(f.name); + --dst->field_count; // revert field count increment + return; + } f.is_binary = 0; f.length = static_cast(value.size()); } // adds a binary field to the ParsedData structure static void add_field_bin(ParsedData* dst, const std::string& name, const uint8_t* data, size_t len) { - auto& f = ensure_slot(dst, name); + if (has_error(dst)) + return; + + if (len == 0) { + add_field_str(dst, name, ""); + return; + } + + auto& f = ensure_slot(dst); + if (has_error(dst)) + return; + + f.name = strdup_safe(name.c_str()); + if (!f.name) { + dst->error_message = strdup_safe("out of memory duplicating field name"); + --dst->field_count; + return; + } + f.value = static_cast(std::malloc(len)); if (!f.value) { dst->error_message = strdup_safe("memory allocation failed for binary field"); + std::free(f.name); + --dst->field_count; return; } std::memcpy(f.value, data, len); @@ -117,11 +169,9 @@ static std::string scalar_to_string(const hilti::rt::type_info::Value& v) { } } -static constexpr int kMaxDepth = 64; // maximum recursion depth for nested structures +static constexpr int kMaxDepth = 64; // maximum recursion depth for nested structures (prevents stack bombs) -// recursively extracts and stores all fields from a HILTI value, -// creates flat field names using dot notation for -// nested structures and bracket notation for array indices +// Recursively flattens HILTI containers to "foo[3].bar" keys and stops at kMaxDepth static void dump_value(ParsedData* dst, const std::string& prefix, const hilti::rt::type_info::Value& v, int depth = 0) { if (depth > kMaxDepth) { add_field_str(dst, prefix, ""); @@ -267,8 +317,9 @@ char** spicy_list_parsers(int* count) { } // parses data using a specified Spicy parser and returns the parsed data +// called only from the single locked OS thread in the Go worker ParsedData* spicy_parse_generic(const char* parser_name, const unsigned char* data, int length) { - if (!parser_name || ! data || length <= 0) + if (!parser_name || !data || length <= 0) return nullptr; auto* res = static_cast(std::calloc(1, sizeof(ParsedData))); diff --git a/protocols/spicy/helpers.go b/protocols/spicy/helpers.go index 307d988..b3bb9c7 100644 --- a/protocols/spicy/helpers.go +++ b/protocols/spicy/helpers.go @@ -22,7 +22,7 @@ func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { switch protocol { case "http": - const maxHTTPBody = 1 << 20 // 1 MiB limit + const maxHTTPBody = 1 << 20 // 1 MiB limit (abuse cap, not HTTP limit) r := bufio.NewReader(conn) raw := make([]byte, 0, 4096) diff --git a/protocols/spicy/parser.go b/protocols/spicy/parser.go index 363afc0..3dfac1c 100644 --- a/protocols/spicy/parser.go +++ b/protocols/spicy/parser.go @@ -45,7 +45,7 @@ func startWorker() { workerOnce.Do(func() { cmdCh = make(chan workerCmd) go func() { - runtime.LockOSThread() // lock to OS thread for C++ runtime thread-local storage + runtime.LockOSThread() // lock to OS thread for C++ runtime thread-local storage (mandatory for Spicy/HILTI TLS) defer runtime.UnlockOSThread() for cmd := range cmdCh { @@ -171,6 +171,14 @@ func Parse(proto string, data []byte) (*ParsedData, error) { select { case raw = <-resp: // normal path case <-time.After(parseTimeout): // worker stalled + // drain resp to free C memory even after we have returned + go func() { + if raw := <-resp; raw != nil { + if p, ok := raw.(*C.ParsedData); ok { + C.spicy_free_parsed_data(p) + } + } + }() return nil, fmt.Errorf("Spicy parse timed-out after %s", parseTimeout) } From 5a1ccd69e6a3671352e0d145b7ea59503e866a7e Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Mon, 7 Jul 2025 05:40:59 +0530 Subject: [PATCH 05/13] feat: adds tests for parser and Spicy http handler --- protocols/spicy/handlers/http_test.go | 153 ++++++++++++++++++++++++++ protocols/spicy/parser_test.go | 126 +++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 protocols/spicy/handlers/http_test.go create mode 100644 protocols/spicy/parser_test.go diff --git a/protocols/spicy/handlers/http_test.go b/protocols/spicy/handlers/http_test.go new file mode 100644 index 0000000..a996e33 --- /dev/null +++ b/protocols/spicy/handlers/http_test.go @@ -0,0 +1,153 @@ +package handlers + +import ( + "bytes" + "context" + "net" + "sync" + "testing" + "time" + + "github.com/mushorg/glutton/connection" + "github.com/mushorg/glutton/protocols/mocks" + "github.com/mushorg/glutton/protocols/spicy" + "github.com/mushorg/glutton/rules" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type mockConn struct { + readBuf *bytes.Buffer + writeBuf *bytes.Buffer + remoteAddr net.Addr + localAddr net.Addr + closed bool +} + +func newMockConn(data string) *mockConn { + return &mockConn{ + readBuf: bytes.NewBufferString(data), + writeBuf: &bytes.Buffer{}, + remoteAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.100"), Port: 12345}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 80}, + } +} + +func (m *mockConn) Read(b []byte) (n int, err error) { return m.readBuf.Read(b) } +func (m *mockConn) Write(b []byte) (n int, err error) { return m.writeBuf.Write(b) } +func (m *mockConn) Close() error { m.closed = true; return nil } +func (m *mockConn) LocalAddr() net.Addr { return m.localAddr } +func (m *mockConn) RemoteAddr() net.Addr { return m.remoteAddr } +func (m *mockConn) SetDeadline(t time.Time) error { return nil } +func (m *mockConn) SetReadDeadline(t time.Time) error { return nil } +func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil } +func (m *mockConn) Written() string { return m.writeBuf.String() } + +func createMockLogger() *mocks.MockLogger { + logger := &mocks.MockLogger{} + + // generic expectations that handle all variations + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything).Return().Maybe() + + return logger +} + +// initialize Spicy once for all tests +var spicyInitOnce sync.Once + +func ensureSpicyInitialized() { + spicyInitOnce.Do(func() { + logger := createMockLogger() + spicy.Initialize(logger) + }) +} + +func TestHandleHTTPBasicGET(t *testing.T) { + ensureSpicyInitialized() + + httpRequest := "GET /test HTTP/1.1\r\nHost: example.com\r\n\r\n" + conn := newMockConn(httpRequest) + + logger := createMockLogger() + honeypot := &mocks.MockHoneypot{} + honeypot.EXPECT().ProduceTCP(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + md := connection.Metadata{ + TargetPort: 80, + Rule: &rules.Rule{Target: "http"}, + } + + ctx := context.Background() + err := HandleHTTP(ctx, conn, md, logger, honeypot) + + require.NoError(t, err) + require.True(t, conn.closed) + + response := conn.Written() + require.Contains(t, response, "HTTP/1.1 200 OK") + + logger.AssertExpectations(t) + honeypot.AssertExpectations(t) +} + +func TestHandleHTTPWithBody(t *testing.T) { + ensureSpicyInitialized() + + httpRequest := "POST /api HTTP/1.1\r\nHost: example.com\r\nContent-Length: 13\r\n\r\n{\"test\":true}" + conn := newMockConn(httpRequest) + + logger := createMockLogger() + honeypot := &mocks.MockHoneypot{} + honeypot.EXPECT().ProduceTCP(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + md := connection.Metadata{TargetPort: 80} + ctx := context.Background() + + err := HandleHTTP(ctx, conn, md, logger, honeypot) + require.NoError(t, err) + require.True(t, conn.closed) + + logger.AssertExpectations(t) + honeypot.AssertExpectations(t) +} + +func TestHandleHTTPMalformedRequest(t *testing.T) { + ensureSpicyInitialized() + + malformedRequest := "GET /path\r\nHost: test\r\n\r\n" + conn := newMockConn(malformedRequest) + + logger := createMockLogger() + honeypot := &mocks.MockHoneypot{} + honeypot.EXPECT().ProduceTCP("spicy-http-failed", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + md := connection.Metadata{TargetPort: 80} + ctx := context.Background() + + err := HandleHTTP(ctx, conn, md, logger, honeypot) + require.Error(t, err) + require.True(t, conn.closed) + + logger.AssertExpectations(t) + honeypot.AssertExpectations(t) +} + +func TestHandleHTTPEmptyRequest(t *testing.T) { + ensureSpicyInitialized() + + conn := newMockConn("") + logger := createMockLogger() + honeypot := &mocks.MockHoneypot{} + + md := connection.Metadata{TargetPort: 80} + ctx := context.Background() + + err := HandleHTTP(ctx, conn, md, logger, honeypot) + require.Error(t, err) // empty request should error (EOF when trying to read) + require.True(t, conn.closed) +} diff --git a/protocols/spicy/parser_test.go b/protocols/spicy/parser_test.go new file mode 100644 index 0000000..9b97617 --- /dev/null +++ b/protocols/spicy/parser_test.go @@ -0,0 +1,126 @@ +package spicy + +import ( + "sync" + "testing" + + "github.com/mushorg/glutton/protocols/mocks" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func createMockLogger() *mocks.MockLogger { + logger := &mocks.MockLogger{} + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Info(mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything, mock.Anything).Return().Maybe() + logger.EXPECT().Error(mock.Anything).Return().Maybe() + return logger +} + +var spicyInitOnce sync.Once +var spicyInitErr error + +func ensureSpicyInitialized() { + spicyInitOnce.Do(func() { + logger := createMockLogger() + spicyInitErr = Initialize(logger) + }) +} + +func TestInitialize(t *testing.T) { + ensureSpicyInitialized() + + logger := createMockLogger() + + err1 := Initialize(logger) + err2 := Initialize(logger) + + if spicyInitErr != nil { + require.Error(t, err1) + require.Error(t, err2) + t.Logf("Initialize failed (expected if Spicy not available): %v", err1) + } else { + require.NoError(t, err1) + require.NoError(t, err2) + } + + logger.AssertExpectations(t) +} + +func TestParseHTTPRequest(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + httpRequest := "GET /test HTTP/1.1\r\nHost: example.com\r\nContent-Length: 4\r\n\r\ntest" + result, err := Parse("http", []byte(httpRequest)) + + if err != nil { + if err.Error() == "no Spicy parser registered for \"http\"" { + t.Skip("HTTP parser not available in this build") + return + } + require.NoError(t, err, "Unexpected parsing error") + } + + require.NotNil(t, result) + require.Equal(t, "http", result.Protocol) + require.NotNil(t, result.Fields) + require.NoError(t, result.Error) + + if method, ok := result.Fields["method"]; ok { + require.Equal(t, "GET", method) + } +} + +func TestParseUnknownProtocol(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + result, err := Parse("unknown-protocol", []byte("test data")) + + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "no Spicy parser registered") +} + +func TestParseEmptyData(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + result, err := Parse("http", []byte{}) + + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "input data is empty") +} + +func TestParseNilData(t *testing.T) { + ensureSpicyInitialized() + + if spicyInitErr != nil { + t.Skipf("Skipping test as Spicy initialization failed: %v", spicyInitErr) + } + + result, err := Parse("http", nil) + + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "input data is empty") +} + +func TestCleanup(t *testing.T) { + err := Cleanup() + require.NoError(t, err) +} From 49b6c58d28f4699b0ae808c32043b4509e8486d8 Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Mon, 7 Jul 2025 15:32:02 +0530 Subject: [PATCH 06/13] chore: tightens up parser tests, adds relevant comments for helpers --- protocols/spicy/helpers.go | 2 ++ protocols/spicy/parser_test.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/protocols/spicy/helpers.go b/protocols/spicy/helpers.go index b3bb9c7..30805f9 100644 --- a/protocols/spicy/helpers.go +++ b/protocols/spicy/helpers.go @@ -78,6 +78,7 @@ func ReadInitialBytes(protocol string, conn net.Conn) ([]byte, error) { // converts a flat map with dot notation keys into a nested map structure. // created initially as a Spicy helper to handle nested data structures +// currently unused, kept for potential future use func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { root := map[string]interface{}{} @@ -131,6 +132,7 @@ func NestedFromFlat(flat map[string]interface{}) map[string]interface{} { } // retrieves a string value from a nested map using a path with dot notation +// currently unused, kept for potential future use func GetDeepStr(m map[string]interface{}, path ...string) string { for _, p := range path { parts := strings.Split(p, ".") diff --git a/protocols/spicy/parser_test.go b/protocols/spicy/parser_test.go index 9b97617..65a0211 100644 --- a/protocols/spicy/parser_test.go +++ b/protocols/spicy/parser_test.go @@ -73,9 +73,14 @@ func TestParseHTTPRequest(t *testing.T) { require.NotNil(t, result.Fields) require.NoError(t, result.Error) - if method, ok := result.Fields["method"]; ok { - require.Equal(t, "GET", method) - } + method := result.Fields["method"] + require.Equal(t, "GET", method) + + uri := result.Fields["uri"] + require.Equal(t, "/test", uri) + + version := result.Fields["version.number"] + require.Equal(t, "1.1", version) } func TestParseUnknownProtocol(t *testing.T) { From 7052c457c423a6c4fdc791e80d25f8eb6f33d46e Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Mon, 7 Jul 2025 18:27:13 +0530 Subject: [PATCH 07/13] refactor: update GitHub Actions to install Spicy C++ runtime, bumps Go version to 1.23 --- .github/workflows/workflow.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 11d6b37..75d835e 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -10,8 +10,22 @@ jobs: checks: runs-on: ubuntu-latest steps: - - name: Dependencies - run: sudo apt install libpcap-dev iptables + - name: Basic Dependencies + run: | + sudo apt-get update + sudo apt-get install -y libpcap-dev iptables zlib1g-dev build-essential + + - name: Install spicyc + run: | + wget https://github.com/zeek/spicy/releases/download/v1.13.1/spicy_linux_ubuntu24.deb + sudo dpkg --install spicy_linux_ubuntu24.deb + rm spicy_linux_ubuntu24.deb + + - name: Install clang17 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 17 - name: Checkout uses: actions/checkout@v3 @@ -19,10 +33,16 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: "^1.21" + go-version: "^1.23" - name: Build + env: + CC: clang + CXX: clang++ run: go build -v ./... - name: Test + env: + CC: clang + CXX: clang++ run: go test -v ./... From 60ca1d6704677d55b9053591aca034ea1aa4d3df Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Mon, 7 Jul 2025 18:30:32 +0530 Subject: [PATCH 08/13] fix: adds CI step for generating Spicy files --- .github/workflows/workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 75d835e..4053b5b 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -35,6 +35,10 @@ jobs: with: go-version: "^1.23" + - name: Build Spicy generated files + run: | + make spicy + - name: Build env: CC: clang From aec7036a7ee71b198eeb9c3e7d0502458cca1b40 Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Mon, 7 Jul 2025 18:38:03 +0530 Subject: [PATCH 09/13] fix: adds Spicy CLI to actions runner path --- .github/workflows/workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 4053b5b..bd3cb86 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -19,8 +19,12 @@ jobs: run: | wget https://github.com/zeek/spicy/releases/download/v1.13.1/spicy_linux_ubuntu24.deb sudo dpkg --install spicy_linux_ubuntu24.deb + sudo apt-get install -f -y # pulling in any missing deps rm spicy_linux_ubuntu24.deb + - name: Add Spicy CLI to PATH + run: echo "/opt/spicy/bin" >> $GITHUB_PATH + - name: Install clang17 run: | wget https://apt.llvm.org/llvm.sh From e933fdbf99b3e44daba563be7d2015c731c910a9 Mon Sep 17 00:00:00 2001 From: Suyash Handa <87384376+Boolean-Autocrat@users.noreply.github.com> Date: Tue, 8 Jul 2025 21:24:59 +0530 Subject: [PATCH 10/13] chore: adds missing ldflags to make build Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2382ade..990178b 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ upx: default: build build: - CC=clang CXX=clang++ go build -o bin/server app/server.go + CC=clang CXX=clang++ go build -ldflags=$(LDFLAGS) -o bin/server app/server.go spicy: cd protocols/spicy && make From 51ed6f28ebd555eb7be31c7dd7ba6d25b4b94f47 Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Tue, 8 Jul 2025 21:41:08 +0530 Subject: [PATCH 11/13] chore: adds error handling for failed file read --- protocols/spicy/handlers/http.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/protocols/spicy/handlers/http.go b/protocols/spicy/handlers/http.go index 47a0f3f..962e7c4 100644 --- a/protocols/spicy/handlers/http.go +++ b/protocols/spicy/handlers/http.go @@ -80,13 +80,16 @@ func handleWallet(uri string, conn net.Conn) bool { return true } -func handleDockerAPIVersion(uri string, conn net.Conn) bool { +func handleDockerAPIVersion(uri string, conn net.Conn, log interfaces.Logger) bool { if !strings.HasPrefix(uri, "/v1.16/version") { return false } - if data, err := tcp.Res.ReadFile("resources/docker_api.json"); err == nil { - conn.Write(append([]byte(fmt.Sprintf( - "HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(data))), data...)) + data, err := tcp.Res.ReadFile("resources/docker_api.json") + if err != nil { + log.Error("failed to read docker_api.json", producer.ErrAttr(err)) + return false + } else { + conn.Write(append([]byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length:%d\r\n\r\n", len(data))), data...)) } return true } @@ -196,7 +199,7 @@ func HandleHTTP(ctx context.Context, conn net.Conn, md connection.Metadata, log handled = handleEthereumRPC(body, conn) || handleYarnNewApplication(method, uri, conn) } - handled = handled || handleWallet(uri, conn) || handleDockerAPIVersion(uri, conn) || handleCitrixSMB(uri, conn) || handleVMwareSend(ctx, body, uri, md, log, hp) + handled = handled || handleWallet(uri, conn) || handleDockerAPIVersion(uri, conn, log) || handleCitrixSMB(uri, conn) || handleVMwareSend(ctx, body, uri, md, log, hp) if !handled { _ = writePlainOK(conn) From a970c55c5b0fd7ed3b8a4b62674183a6866c8277 Mon Sep 17 00:00:00 2001 From: Suyash Handa <87384376+Boolean-Autocrat@users.noreply.github.com> Date: Tue, 8 Jul 2025 22:01:03 +0530 Subject: [PATCH 12/13] refactor: rename inner `raw` variable to `r` in drain goroutine Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- protocols/spicy/parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protocols/spicy/parser.go b/protocols/spicy/parser.go index 3dfac1c..c991071 100644 --- a/protocols/spicy/parser.go +++ b/protocols/spicy/parser.go @@ -173,8 +173,8 @@ func Parse(proto string, data []byte) (*ParsedData, error) { case <-time.After(parseTimeout): // worker stalled // drain resp to free C memory even after we have returned go func() { - if raw := <-resp; raw != nil { - if p, ok := raw.(*C.ParsedData); ok { + if r := <-resp; r != nil { + if p, ok := r.(*C.ParsedData); ok { C.spicy_free_parsed_data(p) } } From 4001b4af84d67ca959dfe2aa1e1ddd03e7a26836 Mon Sep 17 00:00:00 2001 From: Boolean-Autocrat Date: Wed, 9 Jul 2025 17:55:45 +0530 Subject: [PATCH 13/13] refactor: normalizes proto name before parser lookup --- protocols/spicy/parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protocols/spicy/parser.go b/protocols/spicy/parser.go index c991071..2ef747c 100644 --- a/protocols/spicy/parser.go +++ b/protocols/spicy/parser.go @@ -153,12 +153,12 @@ const parseTimeout = 10 * time.Second // the parser is automatically selected based on the protocol name func Parse(proto string, data []byte) (*ParsedData, error) { parsersMutex.RLock() - name, ok := registeredParsers[strings.ToLower(proto)] // parser lookup + key := strings.ToLower(strings.TrimSpace(proto)) + name, ok := registeredParsers[key] // parser lookup parsersMutex.RUnlock() if !ok { return nil, fmt.Errorf("no Spicy parser registered for %q", proto) } - name = strings.TrimSpace(name) if len(data) == 0 { return nil, errors.New("input data is empty")