diff --git a/agent/Cargo.lock b/agent/Cargo.lock index 30208a89cf4..256a3f0f2d8 100644 --- a/agent/Cargo.lock +++ b/agent/Cargo.lock @@ -2473,6 +2473,8 @@ name = "l7" version = "0.1.0" dependencies = [ "bitflags 1.3.2", + "bytes 1.10.1", + "chrono", "serde", ] diff --git a/agent/crates/public/src/l7_protocol.rs b/agent/crates/public/src/l7_protocol.rs index fb5e454e54f..cecedf1f2da 100644 --- a/agent/crates/public/src/l7_protocol.rs +++ b/agent/crates/public/src/l7_protocol.rs @@ -68,6 +68,7 @@ pub enum L7Protocol { MySQL = 60, PostgreSQL = 61, Oracle = 62, + SqlServer = 63, // NoSQL Redis = 80, @@ -144,6 +145,7 @@ impl From for L7Protocol { "webspheremq" => Self::WebSphereMq, "dns" => Self::DNS, "oracle" => Self::Oracle, + "sqlserver" => Self::SqlServer, "iso8583" | "iso-8583" => Self::Iso8583, "triple" => Self::Triple, "tls" => Self::TLS, diff --git a/agent/plugins/l7/Cargo.toml b/agent/plugins/l7/Cargo.toml index 81d020eaa48..cc421b9cc3d 100644 --- a/agent/plugins/l7/Cargo.toml +++ b/agent/plugins/l7/Cargo.toml @@ -7,4 +7,6 @@ edition = "2021" [dependencies] bitflags = "1.3.2" +bytes = "1.10" +chrono = "0.4" serde = { version = "1.0", features = ["derive"] } diff --git a/agent/plugins/l7/src/lib.rs b/agent/plugins/l7/src/lib.rs index 28405e9db91..f16a67154d1 100644 --- a/agent/plugins/l7/src/lib.rs +++ b/agent/plugins/l7/src/lib.rs @@ -15,4 +15,5 @@ */ pub mod some_ip; +pub mod sql_server; pub mod tls; diff --git a/agent/plugins/l7/src/sql_server.rs b/agent/plugins/l7/src/sql_server.rs new file mode 100644 index 00000000000..706ec97a3da --- /dev/null +++ b/agent/plugins/l7/src/sql_server.rs @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 Yunshan Networks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#[derive(Debug, Default)] +pub struct TdsParser { + pub sql: Option, + pub status_code: Option, + pub error_message: Option, + pub affected_row: Option, +} + +impl TdsParser { + pub fn new(_: &[u8]) -> Self { + TdsParser::default() + } + + pub fn parse(&mut self) -> Result<(), ParserError> { + Err(ParserError::InvalidData) + } +} + +#[derive(Debug)] +pub enum ParserError { + IoError(std::io::Error), + UnknownToken(u8), + UnknownEnvType(u8), + InvalidData, + InsufficientData, + Utf8Error(std::string::FromUtf8Error), + Utf16Error(std::string::FromUtf16Error), + UnsupportedDataType, +} + +impl From for ParserError { + fn from(err: std::io::Error) -> Self { + ParserError::IoError(err) + } +} + +impl From for ParserError { + fn from(err: std::string::FromUtf8Error) -> Self { + ParserError::Utf8Error(err) + } +} + +impl From for ParserError { + fn from(err: std::string::FromUtf16Error) -> Self { + ParserError::Utf16Error(err) + } +} diff --git a/agent/src/common/l7_protocol_info.rs b/agent/src/common/l7_protocol_info.rs index aa5287ef14f..4d0bace2978 100644 --- a/agent/src/common/l7_protocol_info.rs +++ b/agent/src/common/l7_protocol_info.rs @@ -31,7 +31,7 @@ use crate::{ fastcgi::FastCGIInfo, pb_adapter::L7ProtocolSendLog, AmqpInfo, BrpcInfo, DnsInfo, DubboInfo, HttpInfo, KafkaInfo, MemcachedInfo, MongoDBInfo, MqttInfo, MysqlInfo, NatsInfo, OpenWireInfo, PingInfo, PostgreInfo, PulsarInfo, RedisInfo, RocketmqInfo, - SofaRpcInfo, TarsInfo, ZmtpInfo, + SofaRpcInfo, SqlServerInfo, TarsInfo, ZmtpInfo, }, AppProtoHead, Result, }, @@ -100,6 +100,7 @@ cfg_if::cfg_if! { SofaRpcInfo(SofaRpcInfo), PingInfo(PingInfo), CustomInfo(CustomInfo), + SqlServerInfo(SqlServerInfo), // add new protocol info below ); } else { @@ -131,6 +132,7 @@ cfg_if::cfg_if! { PingInfo(PingInfo), CustomInfo(CustomInfo), Iso8583Info(crate::flow_generator::protocol_logs::rpc::Iso8583Info), + SqlServerInfo(SqlServerInfo), // add new protocol info below ); } diff --git a/agent/src/common/l7_protocol_log.rs b/agent/src/common/l7_protocol_log.rs index 3923392b1be..b26a2e14039 100644 --- a/agent/src/common/l7_protocol_log.rs +++ b/agent/src/common/l7_protocol_log.rs @@ -44,7 +44,7 @@ use crate::flow_generator::protocol_logs::{ sql::ObfuscateCache, AmqpLog, BrpcLog, DnsLog, DubboLog, HttpLog, KafkaLog, L7ResponseStatus, MemcachedLog, MongoDBLog, MqttLog, MysqlLog, NatsLog, OpenWireLog, PingLog, PostgresqlLog, PulsarLog, - RedisLog, RocketmqLog, SofaRpcLog, TarsLog, ZmtpLog, + RedisLog, RocketmqLog, SofaRpcLog, SqlServerLog, TarsLog, ZmtpLog, }; use crate::flow_generator::Result; @@ -180,6 +180,7 @@ cfg_if::cfg_if! { RocketMQ(RocketmqLog), OpenWire(OpenWireLog), Ping(PingLog), + SqlServer(SqlServerLog), // add protocol below } } @@ -213,6 +214,7 @@ cfg_if::cfg_if! { TLS(crate::flow_generator::protocol_logs::TlsLog), SomeIp(crate::flow_generator::protocol_logs::SomeIpLog), Ping(PingLog), + SqlServer(SqlServerLog), // add protocol below } } diff --git a/agent/src/config/config.rs b/agent/src/config/config.rs index 6d9d6503fc5..2fd5a08b0fd 100644 --- a/agent/src/config/config.rs +++ b/agent/src/config/config.rs @@ -1891,6 +1891,7 @@ impl Default for Filters { ("MySQL".to_string(), "1-65535".to_string()), ("PostgreSQL".to_string(), "1-65535".to_string()), ("Oracle".to_string(), "1521".to_string()), + ("SqlServer".to_string(), "1433".to_string()), ("Redis".to_string(), "1-65535".to_string()), ("MongoDB".to_string(), "1-65535".to_string()), ("Memcached".to_string(), "11211".to_string()), @@ -1923,6 +1924,7 @@ impl Default for Filters { ("MySQL".to_string(), vec![]), ("PostgreSQL".to_string(), vec![]), ("Oracle".to_string(), vec![]), + ("SqlServer".to_string(), vec![]), ("Redis".to_string(), vec![]), ("MongoDB".to_string(), vec![]), ("Memcached".to_string(), vec![]), diff --git a/agent/src/ebpf/README.md b/agent/src/ebpf/README.md index 919b997193d..ee3c1b37478 100644 --- a/agent/src/ebpf/README.md +++ b/agent/src/ebpf/README.md @@ -28,6 +28,7 @@ The following protocols are currently probed: - TLS(handshake) - MONGO - ORACLE +- SqlServer - FASTCGI - ROCKETMQ @@ -104,13 +105,13 @@ end style bpf_tracer_init-2 stroke-width:2px style bpf_tracer_init-3 stroke-width:2px start[tracer start] - start --> enable_ebpf_protocol(1 enable_ebpf_protocol) -.- enable_ebpf_protocol-i([ebable application layer protocols]) + start --> enable_ebpf_protocol(1 enable_ebpf_protocol) -.- enable_ebpf_protocol-i([ebable application layer protocols]) start --> FEATUER(2 set_feature_regex) -.- FEATUER-i([Uprobe OPENSSL/GOLANG filter]) start --> bpf_tracer_init(3 bpf_tracer_init) start --> running_socket_tracer(4 running_socket_tracer) start --> bpf_tracer_finish(5 bpf_tracer_finish)-.-bpf_tracer_finish-i([Indicates that all probes have been set]) - + bpf_tracer_init --> bpf_tracer_init-1(3.1 set bpf_jit_enable,sys_boot_time,max_locked_memory) bpf_tracer_init --> bpf_tracer_init-2(3.2 set log) bpf_tracer_init --> bpf_tracer_init-3(3.3 new thread - ctrl_main) @@ -125,12 +126,12 @@ end running_socket_tracer --> running_socket_tracer-6(4.4 setup_bpf_tracer) running_socket_tracer --> running_socket_tracer-4(4.5 maps_config)-.-running_socket_tracer-4-i([set socket/trace map entries count]) - running_socket_tracer --> running_socket_tracer-5(4.6 set perf buffer reader callback) - running_socket_tracer --> running_socket_tracer-7(4.7 set socket_map_max_reclaim value) + running_socket_tracer --> running_socket_tracer-5(4.6 set perf buffer reader callback) + running_socket_tracer --> running_socket_tracer-7(4.7 set socket_map_max_reclaim value) running_socket_tracer --> running_socket_tracer-8(4.8 tracer_bpf_load)-.-running_socket_tracer-8-i([load ebpf progs and create maps]) running_socket_tracer --> running_socket_tracer-9(4.9 tracer_probes_init)-.-running_socket_tracer-9-i([Call create_probe, prepare all probes before attach/detach]) running_socket_tracer --> running_socket_tracer-10(4.10 update_offset_map_from_btf_vmlinux)-.-running_socket_tracer-10-i([Get the offsets from the BTF files, these offsets are used by the eBPF programs]) - running_socket_tracer-10 -.-> |store member offset to map |members_offset_map[(members_offset map)] + running_socket_tracer-10 -.-> |store member offset to map |members_offset_map[(members_offset map)] running_socket_tracer --> running_socket_tracer-11(4.11 update_proc_info_to_map) -.- running_socket_tracer-11-i([set go/ssl uprobe offsets]) running_socket_tracer-11-.->|store uprobe offsets for the executable file|proc_info_map[(proc_info_map)] running_socket_tracer --> running_socket_tracer-12(4.12 update_protocol_filter_array) @@ -309,7 +310,7 @@ end style collect_ssl_uprobe_syms_from_procfs fill:#ccff,color:#000, stroke-width:2px style IFGETVER stroke-width:2px style collect_ssl_detail stroke-width:2px - style RET fill:#fff,color:#000,stroke:#000,stroke-width:2px + style RET fill:#fff,color:#000,stroke:#000,stroke-width:2px style resolve_bin_file fill:#ccff,color:#000, stroke-width:2px style resolve_and_gen_uprobe_symbol fill:#ccff,color:#000, stroke-width:2px style bcc_elf_foreach_sym fill:#ccff,color:#000, stroke-width:2px @@ -389,7 +390,7 @@ graph LR perf_buffer-.-> Buffer-Reader(Buffer-Reader) subgraph user Buffer-Reader---perf_buffer_read(3 perf_buffer_read)-->reader_event_read(4 reader_event_read)-->reader_raw_cb(5 reader_raw_cb) - end + end ``` ```mermaid @@ -452,7 +453,7 @@ graph LR **Explanation:** -- 7 process_event +- 7 process_event - According to the event_type is EVENT_TYPE_PROC_EXEC or EVENT_TYPE_PROC_EXIT to determine the final call interface. EVENT_TYPE_PROC_EXEC(call go_process_exec(), ssl_process_exec),EVENT_TYPE_PROC_EXIT(call go_process_exit(), ssl_process_exit()). - 8 rust extra events callback - We provide a function that the user can register a callback interface for a specific event. e.g. Use rust function process these events. @@ -478,11 +479,11 @@ graph LR - 4.16.x - 4.16.0, 4.16.1, 4.16.10, 4.16.11, 4.16.12,4.16.13, 4.16.2, 4.16.3, 4.16.4, 4.16.5, 4.16.6, 4.16.7, 4.16.8, 4.16.9 - 4.17.x - - 4.17.0, 4.17.1, 4.17.10, 4.17.11,4.17.12, 4.17.13, 4.17.14, 4.17.2, 4.17.3, 4.17.4, 4.17.5, 4.17.6, 4.17.8, 4.17.9 + - 4.17.0, 4.17.1, 4.17.10, 4.17.11,4.17.12, 4.17.13, 4.17.14, 4.17.2, 4.17.3, 4.17.4, 4.17.5, 4.17.6, 4.17.8, 4.17.9 - 4.18.x - 4.18.0, 4.18.1, 4.18.10,4.18.11, 4.18.12, 4.18.13, 4.18.14, 4.18.15, 4.18.16, 4.18.3, 4.18.4, 4.18.5, 4.18.6, 4.18.7, 4.18.8, 4.18.9 - 4.19.x - - 4.19.0, 4.19.1, 4.19.10, 4.19.11, 4.19.12, 4.19.2, 4.19.3, 4.19.4, 4.19.5, 4.19.6, 4.19.7, 4.19.8, 4.19.9 + - 4.19.0, 4.19.1, 4.19.10, 4.19.11, 4.19.12, 4.19.2, 4.19.3, 4.19.4, 4.19.5, 4.19.6, 4.19.7, 4.19.8, 4.19.9 - 4.20.x - 4.20.0,4.20.1, 4.20.10, 4.20.11, 4.20.12, 4.20.13, 4.20.2, 4.20.3, 4.20.4, 4.20.5, 4.20.6, 4.20.7, 4.20.8 - 5.0.x diff --git a/agent/src/ebpf/kernel/include/common.h b/agent/src/ebpf/kernel/include/common.h index 9a89b5e7f80..5815aef1a5e 100644 --- a/agent/src/ebpf/kernel/include/common.h +++ b/agent/src/ebpf/kernel/include/common.h @@ -80,6 +80,7 @@ enum traffic_protocol { PROTO_MYSQL = 60, PROTO_POSTGRESQL = 61, PROTO_ORACLE = 62, + PROTO_SQL_SERVER = 63, PROTO_REDIS = 80, PROTO_MONGO = 81, PROTO_MEMCACHED = 82, diff --git a/agent/src/ebpf/kernel/include/protocol_inference.h b/agent/src/ebpf/kernel/include/protocol_inference.h index 400f2f04bbf..d77949c9be8 100644 --- a/agent/src/ebpf/kernel/include/protocol_inference.h +++ b/agent/src/ebpf/kernel/include/protocol_inference.h @@ -791,10 +791,10 @@ static __inline enum message_type infer_mysql_message(const char *buf, if (is_socket_info_valid(conn_info->socket_info_ptr)) { /* * Ensure the authentication response packet is captured - * and distinguish it based on the 5th byte (Payload start): + * and distinguish it based on the 5th byte (Payload start): * - * - **Authentication Success (OK Packet):** `0x00` - * - **Authentication Failure (ERR Packet):** `0xFF` + * - **Authentication Success (OK Packet):** `0x00` + * - **Authentication Failure (ERR Packet):** `0xFF` * - **Authentication Switch Request (Auth Switch Request):** `0xFE` */ if (seq <= 1 || (seq == 2 && (com == 0x0 || com == 0xFF || com == 0xFE))) @@ -835,14 +835,14 @@ static __inline enum message_type infer_mysql_message(const char *buf, /* * After establishing a connection, the MySQL server sends a handshake packet. - * The process is as follows: - * - **Server > Client (Handshake Packet)** + * The process is as follows: + * - **Server > Client (Handshake Packet)** * The server sends this handshake packet, which includes the MySQL version, - * thread ID, authentication method, and other information. - * - **Client > Server (Login Request Packet)** + * thread ID, authentication method, and other information. + * - **Client > Server (Login Request Packet)** * The client computes the encrypted password based on `auth-plugin-data` and - * sends it back to the server for verification. - * - **Server > Client (Login Success or Failure)** + * sends it back to the server for verification. + * - **Server > Client (Login Success or Failure)** * The server verifies the client's identity and returns either an **OK Packet** or an **ERR Packet**. * * The handshake packet sent by the server is used for identification. @@ -1228,6 +1228,48 @@ static __inline enum message_type infer_oracle_tns_message(const char *buf, } } +static __inline enum message_type infer_sql_server_message(const char *buf, + size_t count, + struct conn_info_s + *conn_info) +{ +#define HEADER_SIZE 8 +#define MESSAGE_TYPE_OFFSET 0 +#define LENGTH_OFFSET 1 +#define WINDOWS_OFFSET 7 +#define MESSAGE_TYPE_SQL_BATCH 1 +#define MESSAGE_TYPE_RPC 3 +#define MESSAGE_TYPE_RESPONSE 4 + if (!protocol_port_check_1(PROTO_SQL_SERVER, conn_info)) + return MSG_UNKNOWN; + if (conn_info->tuple.l4_protocol != IPPROTO_TCP || count < HEADER_SIZE) { + return MSG_UNKNOWN; + } + + if (is_infer_socket_valid(conn_info->socket_info_ptr)) { + if (conn_info->socket_info_ptr->l7_proto != PROTO_SQL_SERVER) + return MSG_UNKNOWN; + } + + __u8 pkt_type = buf[MESSAGE_TYPE_OFFSET]; + __u16 pkt_length = __bpf_ntohs(*(__u16 *) & buf[LENGTH_OFFSET]); + __u8 windows = buf[WINDOWS_OFFSET]; + if (pkt_length < HEADER_SIZE || windows != 0) { + return MSG_UNKNOWN; + } + + if (pkt_type == MESSAGE_TYPE_RESPONSE) { + return MSG_RESPONSE; + } + if (pkt_type == MESSAGE_TYPE_SQL_BATCH) { + return MSG_REQUEST; + } + if (pkt_type == MESSAGE_TYPE_RPC) { + return MSG_REQUEST; + } + return MSG_UNKNOWN; +} + // https://en.wikipedia.org/wiki/ISO_8583 static __inline enum message_type infer_iso8583_message(const char *buf, size_t count, @@ -1581,11 +1623,11 @@ static __inline enum message_type infer_dns_message(const char *buf, bool update_tcp_dns_prev_count = false; struct dns_header *dns = (struct dns_header *)buf; - + /* * Note that TCP DNS adds two length bytes at the beginning of the protocol, * whereas UDP DNS does not. We need to handle this properly to ensure that - * these two length bytes are not sent to the upper layer. + * these two length bytes are not sent to the upper layer. * * When receiving data, the client does not first receive two bytes but instead * receives everything at once; whereas the server receives two bytes (length) first @@ -4286,7 +4328,7 @@ infer_protocol_1(struct ctx_info_s *ctx, * encrypted data can be discarded to prevent it from being involved * in subsequent protocol inference, thereby avoiding performance * degradation. - */ + */ if (is_socket_info_valid(conn_info->socket_info_ptr)) { if (conn_info->socket_info_ptr->is_tls && !skip_http2_kprobe()) @@ -4483,6 +4525,15 @@ infer_protocol_1(struct ctx_info_s *ctx, return inferred_message; } break; + case PROTO_SQL_SERVER: + if ((inferred_message.type = + infer_sql_server_message(infer_buf, count, + conn_info)) != + MSG_UNKNOWN) { + inferred_message.protocol = PROTO_SQL_SERVER; + return inferred_message; + } + break; case PROTO_ISO8583: if ((inferred_message.type = infer_iso8583_message(infer_buf, count, @@ -4633,6 +4684,14 @@ infer_protocol_1(struct ctx_info_s *ctx, if (inferred_message.type == MSG_PRESTORE) return inferred_message; inferred_message.protocol = PROTO_MYSQL; +#if defined(LINUX_VER_KFUNC) || defined(LINUX_VER_5_2_PLUS) + } else if (skip_proto != PROTO_SQL_SERVER && (inferred_message.type = +#else + } else if ((inferred_message.type = +#endif + infer_sql_server_message(infer_buf, count, + conn_info)) != MSG_UNKNOWN) { + inferred_message.protocol = PROTO_SQL_SERVER; #if defined(LINUX_VER_KFUNC) || defined(LINUX_VER_5_2_PLUS) } else if (skip_proto != PROTO_FASTCGI && (inferred_message.type = #else diff --git a/agent/src/ebpf/mod.rs b/agent/src/ebpf/mod.rs index e47bb80995c..045707d03ba 100644 --- a/agent/src/ebpf/mod.rs +++ b/agent/src/ebpf/mod.rs @@ -75,6 +75,8 @@ pub const SOCK_DATA_POSTGRESQL: u16 = 61; #[allow(dead_code)] pub const SOCK_DATA_ORACLE: u16 = 62; #[allow(dead_code)] +pub const SOCK_DATA_SQL_SERVER: u16 = 63; +#[allow(dead_code)] pub const SOCK_DATA_REDIS: u16 = 80; #[allow(dead_code)] pub const SOCK_DATA_MONGO: u16 = 81; diff --git a/agent/src/ebpf/samples/rust/socket-tracer/src/main.rs b/agent/src/ebpf/samples/rust/socket-tracer/src/main.rs index 5eb4d1bc905..a05200bb42c 100644 --- a/agent/src/ebpf/samples/rust/socket-tracer/src/main.rs +++ b/agent/src/ebpf/samples/rust/socket-tracer/src/main.rs @@ -248,6 +248,8 @@ extern "C" fn socket_trace_callback(_: *mut c_void, queue_id: c_int, sd: *mut SK proto_tag.push_str("TLS"); } else if sk_proto_safe(sd) == SOCK_DATA_ORACLE { proto_tag.push_str("ORACLE"); + } else if sk_proto_safe(sd) == SOCK_DATA_SQL_SERVER { + proto_tag.push_str("SQLSERVER"); } else if sk_proto_safe(sd) == SOCK_DATA_OPENWIRE { proto_tag.push_str("OPENWIRE"); } else if sk_proto_safe(sd) == SOCK_DATA_ZMTP { diff --git a/agent/src/ebpf/user/ctrl_tracer.c b/agent/src/ebpf/user/ctrl_tracer.c index 51161bb0271..1c01a89df19 100644 --- a/agent/src/ebpf/user/ctrl_tracer.c +++ b/agent/src/ebpf/user/ctrl_tracer.c @@ -150,6 +150,7 @@ static void datadump_help(void) fprintf(stderr, " 60: PROTO_MYSQL\n"); fprintf(stderr, " 61: PROTO_POSTGRESQL\n"); fprintf(stderr, " 62: PROTO_ORACLE\n"); + fprintf(stderr, " 63: PROTO_SQL_SERVER\n"); fprintf(stderr, " 80: PROTO_REDIS\n"); fprintf(stderr, " 81: PROTO_MONGO\n"); fprintf(stderr, " 82: PROTO_MEMCACHED\n"); diff --git a/agent/src/ebpf/user/socket.h b/agent/src/ebpf/user/socket.h index b09799fce57..f8324be00b2 100644 --- a/agent/src/ebpf/user/socket.h +++ b/agent/src/ebpf/user/socket.h @@ -207,7 +207,7 @@ struct bpf_socktrace_params { * Socket Information * For detailed field descriptions, see the comments * in 'struct socket_info_s'. - */ + */ uint64_t socket_id; uint64_t seq; uint16_t l7_proto; @@ -227,7 +227,7 @@ struct bpf_socktrace_params { /* * Additional (monitoring) information for the socket * trace function module. - */ + */ uint8_t tracer_state; uint32_t kern_socket_map_max; uint32_t kern_socket_map_used; @@ -297,6 +297,8 @@ static inline char *get_proto_name(uint16_t proto_id) return "PgSQL"; case PROTO_ORACLE: return "Oracle"; + case PROTO_SQL_SERVER: + return "SqlServer"; case PROTO_FASTCGI: return "FastCGI"; case PROTO_BRPC: @@ -454,15 +456,15 @@ void uprobe_match_pid_handle(int feat, int pid, enum match_pids_act act); /** * @brief Disables the KPROBE feature while retaining UPROBE and I/O event handling. * - * This function will disable the KPROBE functionality, but UPROBE and I/O event processing - * will continue to work as usual. + * This function will disable the KPROBE functionality, but UPROBE and I/O event processing + * will continue to work as usual. */ void disable_kprobe_feature(void); /** * @brief Enables the KPROBE feature. * - * This function enables the KPROBE functionality, allowing kernel probes to be used + * This function enables the KPROBE functionality, allowing kernel probes to be used * for monitoring and tracing specific points in the kernel. */ void enable_kprobe_feature(void); diff --git a/agent/src/flow_generator/protocol_logs.rs b/agent/src/flow_generator/protocol_logs.rs index 49caac0e86f..8581617fae5 100644 --- a/agent/src/flow_generator/protocol_logs.rs +++ b/agent/src/flow_generator/protocol_logs.rs @@ -42,7 +42,7 @@ pub use rpc::{ }; pub use sql::{ MemcachedInfo, MemcachedLog, MongoDBInfo, MongoDBLog, MysqlInfo, MysqlLog, PostgreInfo, - PostgresqlLog, RedisInfo, RedisLog, + PostgresqlLog, RedisInfo, RedisLog, SqlServerInfo, SqlServerLog, }; cfg_if::cfg_if! { diff --git a/agent/src/flow_generator/protocol_logs/sql/mod.rs b/agent/src/flow_generator/protocol_logs/sql/mod.rs index a68f4b35cb2..c65d338c09b 100644 --- a/agent/src/flow_generator/protocol_logs/sql/mod.rs +++ b/agent/src/flow_generator/protocol_logs/sql/mod.rs @@ -31,12 +31,14 @@ mod postgresql; mod redis; mod sql_check; mod sql_obfuscate; +mod sql_server; pub use memcached::{MemcachedInfo, MemcachedLog}; pub use mongo::{MongoDBInfo, MongoDBLog}; pub use mysql::{MysqlInfo, MysqlLog}; pub use postgresql::{PostgreInfo, PostgresqlLog}; pub use redis::{RedisInfo, RedisLog}; +pub use sql_server::{SqlServerInfo, SqlServerLog}; cfg_if::cfg_if! { if #[cfg(feature = "enterprise")] { diff --git a/agent/src/flow_generator/protocol_logs/sql/sql_server.rs b/agent/src/flow_generator/protocol_logs/sql/sql_server.rs new file mode 100644 index 00000000000..63f0b1f7e3a --- /dev/null +++ b/agent/src/flow_generator/protocol_logs/sql/sql_server.rs @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2024 Yunshan Networks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use std::fmt; + +use serde::Serialize; + +use crate::{ + common::{ + enums::IpProtocol, + flow::{L7PerfStats, L7Protocol, PacketDirection}, + l7_protocol_info::{L7ProtocolInfo, L7ProtocolInfoInterface}, + l7_protocol_log::{L7ParseResult, L7ProtocolParserInterface, LogCache, ParseParam}, + meta_packet::ApplicationFlags, + }, + config::handler::LogParserConfig, + flow_generator::{ + error::Result, + protocol_logs::{ + pb_adapter::{L7ProtocolSendLog, L7Request, L7Response}, + set_captured_byte, + sql::ObfuscateCache, + value_is_default, AppProtoHead, L7ResponseStatus, + }, + }, +}; +use l7::sql_server::TdsParser; +use public::l7_protocol::LogMessageType; + +#[derive(Serialize, Debug, Default, Clone)] +pub struct SqlServerInfo { + msg_type: LogMessageType, + #[serde(skip)] + is_tls: bool, + + #[serde(rename = "request_resource", skip_serializing_if = "value_is_default")] + pub sql: String, + #[serde(skip_serializing_if = "value_is_default")] + pub status_code: i32, + pub error_message: String, + pub affected_row: u64, + pub resp_status: L7ResponseStatus, + + captured_request_byte: u32, + captured_response_byte: u32, + + rrt: u64, + + #[serde(skip)] + is_on_blacklist: bool, +} + +impl L7ProtocolInfoInterface for SqlServerInfo { + fn session_id(&self) -> Option { + None + } + + fn merge_log(&mut self, other: &mut L7ProtocolInfo) -> Result<()> { + if let L7ProtocolInfo::SqlServerInfo(other) = other { + return self.merge(other); + } + Ok(()) + } + + fn app_proto_head(&self) -> Option { + Some(AppProtoHead { + proto: L7Protocol::SqlServer, + msg_type: self.msg_type, + rrt: self.rrt, + }) + } + + fn is_tls(&self) -> bool { + self.is_tls + } + + fn get_request_resource_length(&self) -> usize { + self.sql.len() + } + + fn is_on_blacklist(&self) -> bool { + self.is_on_blacklist + } +} + +impl SqlServerInfo { + pub fn merge(&mut self, other: &mut Self) -> Result<()> { + std::mem::swap(&mut self.status_code, &mut other.status_code); + std::mem::swap(&mut self.error_message, &mut other.error_message); + std::mem::swap(&mut self.affected_row, &mut other.affected_row); + self.resp_status = other.resp_status; + self.captured_response_byte = other.captured_response_byte; + if other.is_on_blacklist { + self.is_on_blacklist = other.is_on_blacklist; + } + Ok(()) + } + + fn set_is_on_blacklist(&mut self, config: &LogParserConfig) { + if let Some(t) = config.l7_log_blacklist_trie.get(&L7Protocol::SqlServer) { + self.is_on_blacklist = t.request_resource.is_on_blacklist(&self.sql.as_str()) + } + } +} + +impl fmt::Display for SqlServerInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SqlServerInfo {{ sql: {:?}, ", self.sql)?; + write!(f, "status_code: {:?}, ", &self.status_code)?; + write!(f, "error_message: {:?}, ", &self.error_message)?; + write!(f, "affected_row: {:?} }}", &self.affected_row) + } +} + +impl From for L7ProtocolSendLog { + fn from(f: SqlServerInfo) -> Self { + let flags = if f.is_tls { + ApplicationFlags::TLS.bits() + } else { + ApplicationFlags::NONE.bits() + }; + let log = L7ProtocolSendLog { + captured_request_byte: f.captured_request_byte, + captured_response_byte: f.captured_response_byte, + req: L7Request { + resource: f.sql, + ..Default::default() + }, + resp: L7Response { + status: f.resp_status, + exception: f.error_message, + code: if f.status_code > 0 { + Some(f.status_code) + } else { + None + }, + ..Default::default() + }, + row_effect: f.affected_row as u32, + flags, + ..Default::default() + }; + return log; + } +} + +impl From<&SqlServerInfo> for LogCache { + fn from(info: &SqlServerInfo) -> Self { + LogCache { + msg_type: info.msg_type, + resp_status: info.resp_status, + on_blacklist: info.is_on_blacklist, + ..Default::default() + } + } +} + +#[derive(Default)] +pub struct SqlServerLog { + has_request: bool, + perf_stats: Vec, + obfuscate: bool, +} + +impl L7ProtocolParserInterface for SqlServerLog { + fn check_payload(&mut self, payload: &[u8], param: &ParseParam) -> Option { + if !param.ebpf_type.is_raw_protocol() { + return None; + } + if param.l4_protocol != IpProtocol::TCP { + return None; + } + + let mut tds = TdsParser::new(payload.into()); + + tds.parse().ok(); + + if tds.sql.is_some() { + return Some(LogMessageType::Request); + } + + None + } + + fn parse_payload(&mut self, payload: &[u8], param: &ParseParam) -> Result { + self.perf_stats.clear(); + let mut info = SqlServerInfo::default(); + info.is_tls = param.is_tls(); + let mut tds = TdsParser::new(payload.into()); + + tds.parse().ok(); + + match param.direction { + PacketDirection::ClientToServer => { + let Some(sql) = tds.sql else { + return Ok(L7ParseResult::None); + }; + info.msg_type = LogMessageType::Request; + info.sql = sql; + self.has_request = true; + } + PacketDirection::ServerToClient if self.has_request => { + if tds.affected_row.is_none() + && tds.status_code.is_none() + && tds.error_message.is_none() + { + return Ok(L7ParseResult::None); + }; + + if let Some(affected_row) = tds.affected_row { + info.affected_row = affected_row; + } + if let Some(status_code) = tds.status_code { + info.status_code = status_code; + } + if let Some(error_message) = tds.error_message { + info.error_message = error_message; + info.resp_status = L7ResponseStatus::ClientError; + } else { + info.resp_status = L7ResponseStatus::Ok; + } + info.msg_type = LogMessageType::Response; + + self.has_request = false; + } + _ => return Ok(L7ParseResult::None), + } + + set_captured_byte!(info, param); + if let Some(config) = param.parse_config { + info.set_is_on_blacklist(config); + } + + if param.parse_perf { + let mut perf_stat = L7PerfStats::default(); + if let Some(stats) = info.perf_stats(param) { + info.rrt = stats.rrt_sum; + perf_stat.sequential_merge(&stats); + } + self.perf_stats.push(perf_stat); + }; + + if param.parse_log { + Ok(L7ParseResult::Single(L7ProtocolInfo::SqlServerInfo(info))) + } else { + Ok(L7ParseResult::None) + } + } + + fn protocol(&self) -> L7Protocol { + L7Protocol::SqlServer + } + + fn parsable_on_udp(&self) -> bool { + false + } + + fn perf_stats(&mut self) -> Vec { + std::mem::take(&mut self.perf_stats) + } + + fn set_obfuscate_cache(&mut self, obfuscate_cache: Option) { + self.obfuscate = obfuscate_cache.is_some(); + } +} + +impl SqlServerLog { + fn reset(&mut self) { + self.perf_stats.clear(); + } +} diff --git a/server/agent_config/README-CH.md b/server/agent_config/README-CH.md index e84ed2b515d..2c178ab98a4 100644 --- a/server/agent_config/README-CH.md +++ b/server/agent_config/README-CH.md @@ -7886,6 +7886,7 @@ processors: RocketMQ: 1-65535 SofaRPC: 1-65535 SomeIP: 1-65535 + SqlServer: 1433 TLS: 443,6443 Tars: 1-65535 WebSphereMQ: 1-65535 @@ -7914,7 +7915,7 @@ HTTP2: 1-65535 注意： 1. 该参数中，HTTP2 和 TLS 协议的配置仅对 Kprobe 有效，对 Uprobe 无效； - 支持协议：[https://www.deepflow.io/docs/zh/features/l7-protocols/overview/](https://www.deepflow.io/docs/zh/features/l7-protocols/overview/) - - Oracle 和 TLS 仅在企业版中支持。 + - Oracle、SqlServer 和 TLS 仅在企业版中支持。 2. 如需控制 `gRPC` 协议，请使用 `HTTP2` 配置。 #### Tag 过滤器 {#processors.request_log.filters.tag_filters} @@ -7958,6 +7959,7 @@ processors: RocketMQ: [] SOFARPC: [] SomeIP: [] + SqlServer: [] TLS: [] Tars: [] WebSphereMQ: [] @@ -8039,7 +8041,7 @@ blacklist. Including business request logs might lead to breaks in the distribut Supported protocols: https://www.deepflow.io/docs/features/l7-protocols/overview/ -Oracle and TLS is only supported in the Enterprise Edition. +Oracle, SqlServer and TLS is only supported in the Enterprise Edition. ###### 字段名 {#processors.request_log.filters.tag_filters.HTTP.field_name} diff --git a/server/agent_config/README.md b/server/agent_config/README.md index b5ce0a5adf8..3f7b45515da 100644 --- a/server/agent_config/README.md +++ b/server/agent_config/README.md @@ -7630,7 +7630,7 @@ processors: Turning off some protocol identification can reduce deepflow-agent resource consumption. Supported protocols: [https://www.deepflow.io/docs/features/l7-protocols/overview/](https://www.deepflow.io/docs/features/l7-protocols/overview/) -Oracle and TLS is only supported in the Enterprise Edition. +Oracle, SqlServer and TLS is only supported in the Enterprise Edition. #### Protocol Special Config {#processors.request_log.application_protocol_inference.protocol_special_config} @@ -8060,6 +8060,7 @@ processors: RocketMQ: 1-65535 SofaRPC: 1-65535 SomeIP: 1-65535 + SqlServer: 1433 TLS: 443,6443 Tars: 1-65535 WebSphereMQ: 1-65535 @@ -8089,7 +8090,7 @@ NOTE: 1. HTTP2 and TLS are only used for Kprobe, not applicable to Uprobe. All data obtained through Uprobe is not subject to port restrictions. - Supported protocols: [https://www.deepflow.io/docs/features/l7-protocols/overview/](https://www.deepflow.io/docs/features/l7-protocols/overview/) - - Oracle and TLS is only supported in the Enterprise Edition. + - Oracle, SqlServer and TLS is only supported in the Enterprise Edition. 2. Attention: use `HTTP2` for `gRPC` Protocol. #### Tag Filters {#processors.request_log.filters.tag_filters} @@ -8133,6 +8134,7 @@ processors: RocketMQ: [] SOFARPC: [] SomeIP: [] + SqlServer: [] TLS: [] Tars: [] WebSphereMQ: [] @@ -8173,7 +8175,7 @@ blacklist. Including business request logs might lead to breaks in the distribut Supported protocols: [https://www.deepflow.io/docs/features/l7-protocols/overview/](https://www.deepflow.io/docs/features/l7-protocols/overview/) -Oracle and TLS is only supported in the Enterprise Edition. +Oracle, SqlServer and TLS is only supported in the Enterprise Edition. ##### $HTTP Tag Filters {#processors.request_log.filters.tag_filters.HTTP} @@ -8221,7 +8223,7 @@ blacklist. Including business request logs might lead to breaks in the distribut Supported protocols: https://www.deepflow.io/docs/features/l7-protocols/overview/ -Oracle and TLS is only supported in the Enterprise Edition. +Oracle, SqlServer and TLS is only supported in the Enterprise Edition. ###### Field Name {#processors.request_log.filters.tag_filters.HTTP.field_name} diff --git a/server/agent_config/template.yaml b/server/agent_config/template.yaml index 1506c178cb1..1ef112a6ebc 100644 --- a/server/agent_config/template.yaml +++ b/server/agent_config/template.yaml @@ -5254,7 +5254,7 @@ processors: # en: |- # Turning off some protocol identification can reduce deepflow-agent resource consumption. # Supported protocols: [https://www.deepflow.io/docs/features/l7-protocols/overview/](https://www.deepflow.io/docs/features/l7-protocols/overview/) - # Oracle and TLS is only supported in the Enterprise Edition. + # Oracle, SqlServer and TLS is only supported in the Enterprise Edition. # ch: |- # deepflow-agent 仅对列表内的应用协议进行数据采集。通过该参数可以控制 agent 的数据采集范围以 # 降低资源消耗。 @@ -5624,7 +5624,7 @@ processors: # 1. HTTP2 and TLS are only used for Kprobe, not applicable to Uprobe. # All data obtained through Uprobe is not subject to port restrictions. # - Supported protocols: [https://www.deepflow.io/docs/features/l7-protocols/overview/](https://www.deepflow.io/docs/features/l7-protocols/overview/) - # - Oracle and TLS is only supported in the Enterprise Edition. + # - Oracle, SqlServer and TLS is only supported in the Enterprise Edition. # 2. Attention: use `HTTP2` for `gRPC` Protocol. # ch: |- # 配置样例: @@ -5636,7 +5636,7 @@ processors: # 注意： # 1. 该参数中，HTTP2 和 TLS 协议的配置仅对 Kprobe 有效，对 Uprobe 无效； # - 支持协议：[https://www.deepflow.io/docs/zh/features/l7-protocols/overview/](https://www.deepflow.io/docs/zh/features/l7-protocols/overview/) - # - Oracle 和 TLS 仅在企业版中支持。 + # - Oracle、SqlServer 和 TLS 仅在企业版中支持。 # 2. 如需控制 `gRPC` 协议，请使用 `HTTP2` 配置。 # upgrade_from: static_config.l7-protocol-ports port_number_prefilters: @@ -5651,6 +5651,7 @@ processors: MySQL: 1-65535 PostgreSQL: 1-65535 Oracle: 1521 + SqlServer: 1433 Redis: 1-65535 MongoDB: 1-65535 Memcached: 11211 @@ -5699,7 +5700,7 @@ processors: # # Supported protocols: [https://www.deepflow.io/docs/features/l7-protocols/overview/](https://www.deepflow.io/docs/features/l7-protocols/overview/) # - # Oracle and TLS is only supported in the Enterprise Edition. + # Oracle, SqlServer and TLS is only supported in the Enterprise Edition. # ch: |- # 控制不同应用协议数据采集时的 Tag。协议名不区分大小写。 # Tag filter 配置例子: @@ -5746,7 +5747,7 @@ processors: # # Supported protocols: https://www.deepflow.io/docs/features/l7-protocols/overview/ # - # Oracle and TLS is only supported in the Enterprise Edition. + # Oracle, SqlServer and TLS is only supported in the Enterprise Edition. # upgrade_from: static_config.l7-log-blacklist.$protocol # --- # type: string @@ -5814,6 +5815,7 @@ processors: MySQL: [] PostgreSQL: [] Oracle: [] + SqlServer: [] Redis: [] MongoDB: [] Memcached: [] diff --git a/server/libs/datatype/flow.go b/server/libs/datatype/flow.go index cfd357f2b9b..7754d4241be 100644 --- a/server/libs/datatype/flow.go +++ b/server/libs/datatype/flow.go @@ -163,6 +163,7 @@ const ( L7_PROTOCOL_MYSQL L7Protocol = 60 L7_PROTOCOL_POSTGRE L7Protocol = 61 L7_PROTOCOL_ORACLE L7Protocol = 62 + L7_PROTOCOL_SQL_SERVER L7Protocol = 63 L7_PROTOCOL_REDIS L7Protocol = 80 L7_PROTOCOL_MONGODB L7Protocol = 81 L7_PROTOCOL_MEMCACHED L7Protocol = 82 @@ -692,6 +693,12 @@ func (p L7Protocol) String(isTLS bool) string { } else { return "Oracle" } + case L7_PROTOCOL_SQL_SERVER: + if isTLS { + return "SqlServer_TLS" + } else { + return "SqlServer" + } case L7_PROTOCOL_ISO8583: if isTLS { return "ISO-8583_TLS" @@ -808,6 +815,7 @@ var L7ProtocolStringMap = map[string]L7Protocol{ strings.ToLower(L7_PROTOCOL_MYSQL.String(false)): L7_PROTOCOL_MYSQL, strings.ToLower(L7_PROTOCOL_POSTGRE.String(false)): L7_PROTOCOL_POSTGRE, strings.ToLower(L7_PROTOCOL_ORACLE.String(false)): L7_PROTOCOL_ORACLE, + strings.ToLower(L7_PROTOCOL_SQL_SERVER.String(false)): L7_PROTOCOL_SQL_SERVER, strings.ToLower(L7_PROTOCOL_ISO8583.String(false)): L7_PROTOCOL_ISO8583, strings.ToLower(L7_PROTOCOL_TRIPLE.String(false)): L7_PROTOCOL_TRIPLE, strings.ToLower(L7_PROTOCOL_REDIS.String(false)): L7_PROTOCOL_REDIS, diff --git a/server/querier/db_descriptions/clickhouse/tag/enum/l7_protocol b/server/querier/db_descriptions/clickhouse/tag/enum/l7_protocol index 74f067c7a08..4a6640df0e6 100644 --- a/server/querier/db_descriptions/clickhouse/tag/enum/l7_protocol +++ b/server/querier/db_descriptions/clickhouse/tag/enum/l7_protocol @@ -14,6 +14,7 @@ 60 , MySQL , 61 , PostgreSQL , 62 , Oracle , +63 , SqlServer , 80 , Redis , 81 , MongoDB , 82 , Memcached ,