diff --git a/src/content/docs/guides/collecting/get-data-from-the-network.mdx b/src/content/docs/guides/collecting/get-data-from-the-network.mdx index d5007f5fa..6b16f9116 100644 --- a/src/content/docs/guides/collecting/get-data-from-the-network.mdx +++ b/src/content/docs/guides/collecting/get-data-from-the-network.mdx @@ -61,11 +61,11 @@ high-volume, loss-tolerant data like syslog messages or metrics. ### Receive UDP datagrams -Use from_udp to receive UDP messages as +Use accept_udp to receive UDP messages as structured events: ```tql -from_udp "0.0.0.0:514" +accept_udp "0.0.0.0:514" ``` Each datagram becomes an event with `data` (the message content) and `peer` @@ -76,7 +76,7 @@ Each datagram becomes an event with `data` (the message content) and `peer` A common pattern is receiving syslog over UDP: ```tql -from_udp "0.0.0.0:514" +accept_udp "0.0.0.0:514" this = data.parse_syslog() ``` @@ -85,7 +85,7 @@ this = data.parse_syslog() Include the sender's IP address and collection timestamp in your events: ```tql -from_udp "0.0.0.0:514" +accept_udp "0.0.0.0:514" syslog = data.parse_syslog() this = { ...syslog, diff --git a/src/content/docs/guides/collecting/index.mdx b/src/content/docs/guides/collecting/index.mdx index c2b683232..b727ff0ce 100644 --- a/src/content/docs/guides/collecting/index.mdx +++ b/src/content/docs/guides/collecting/index.mdx @@ -13,7 +13,7 @@ TQL provides two types of input operators: a [subpipeline](/reference/programs/#parsing-subpipelines). - **Direct event operators** like from_kafka - and from_udp produce structured events + and accept_udp produce structured events directly without an intermediate byte stream. :::caution[Migration note] @@ -78,7 +78,7 @@ interfaces: ```tql // UDP syslog receiver -from_udp "0.0.0.0:514" +accept_udp "0.0.0.0:514" // TCP with TLS from "tcp://0.0.0.0:8443", tls=true diff --git a/src/content/docs/integrations/syslog.mdx b/src/content/docs/integrations/syslog.mdx index 31a7a47b2..b4ff52528 100644 --- a/src/content/docs/integrations/syslog.mdx +++ b/src/content/docs/integrations/syslog.mdx @@ -22,10 +22,10 @@ Together, these building blocks enable round-trip Syslog processing. ### Create a Syslog Server To receive Syslog messages on a UDP socket, use -from_udp: +accept_udp: ```tql -from_udp "0.0.0.0:514" +accept_udp "0.0.0.0:514" this = data.parse_syslog() publish "syslog" ``` @@ -201,7 +201,7 @@ from { message: " PARENT process running...", } write_syslog -save_udp "1.2.3.4:514" +to_udp "1.2.3.4:514" ``` This pipeline sends the following RFC 5424-formatted message to diff --git a/src/content/docs/integrations/udp.mdx b/src/content/docs/integrations/udp.mdx index 70f8dcd33..3de905869 100644 --- a/src/content/docs/integrations/udp.mdx +++ b/src/content/docs/integrations/udp.mdx @@ -14,14 +14,14 @@ Use the IP address `0.0.0.0` to listen on all available network interfaces. ## Examples -Use from_udp to receive UDP datagrams as -structured events containing message data and peer information. For sending, use -save_udp with a write operator. +Use accept_udp to receive UDP datagrams as +structured events containing message data and peer information. Use to_udp +to send one UDP datagram per event directly from structured data. ### Receive syslog messages over UDP ```tql -from_udp "0.0.0.0:514" +accept_udp "0.0.0.0:514" this = data.parse_syslog() ``` @@ -29,6 +29,5 @@ this = data.parse_syslog() ```tql from {message: "Tenzir"} -write_ndjson -save_udp "1.2.3.4:8080" +to_udp "1.2.3.4:8080", message=message ``` diff --git a/src/content/docs/reference/operators.mdx b/src/content/docs/reference/operators.mdx index d87882d0c..4a6cb79a8 100644 --- a/src/content/docs/reference/operators.mdx +++ b/src/content/docs/reference/operators.mdx @@ -391,10 +391,10 @@ operators: description: 'Retrieves PowerQuery results from SentinelOne Singularity Data Lake.' example: 'from_sentinelone_data_lake "https://…", …' path: 'reference/operators/from_sentinelone_data_lake' - - name: 'from_udp' + - name: 'accept_udp' description: 'Receives UDP datagrams and outputs structured events.' - example: 'from_udp "0.0.0.0:8090"' - path: 'reference/operators/from_udp' + example: 'accept_udp "0.0.0.0:8090"' + path: 'reference/operators/accept_udp' - name: 'from_velociraptor' description: 'Submits VQL to a Velociraptor server and returns the response as events.' example: 'from_velociraptor subscribe="Windows"' @@ -451,10 +451,6 @@ operators: description: 'Loads bytes from a TCP or TLS connection.' example: 'load_tcp "0.0.0.0:8090" { read_json }' path: 'reference/operators/load_tcp' - - name: 'load_udp' - description: 'Loads bytes from a UDP socket.' - example: 'load_udp "0.0.0.0:8090"' - path: 'reference/operators/load_udp' - name: 'load_zmq' description: 'Receives ZeroMQ messages.' example: 'load_zmq' @@ -691,10 +687,10 @@ operators: description: 'Saves bytes to a TCP or TLS connection.' example: 'save_tcp "0.0.0.0:8090", tls=true' path: 'reference/operators/save_tcp' - - name: 'save_udp' - description: 'Saves bytes to a UDP socket.' - example: 'save_udp "0.0.0.0:8090"' - path: 'reference/operators/save_udp' + - name: 'to_udp' + description: 'Sends one UDP datagram per input event.' + example: 'to_udp "127.0.0.1:514"' + path: 'reference/operators/to_udp' - name: 'save_zmq' description: 'Sends bytes as ZeroMQ messages.' example: 'save_zmq' @@ -2134,14 +2130,6 @@ load_tcp "0.0.0.0:8090" { read_json } - - -```tql -load_udp "0.0.0.0:8090" -``` - - - ```tql @@ -2269,10 +2257,10 @@ from_sentinelone_data_lake "https://…", … - + ```tql -from_udp "0.0.0.0:8090" +accept_udp "0.0.0.0:8090" ``` @@ -2490,10 +2478,10 @@ save_tcp "0.0.0.0:8090", tls=true - + ```tql -save_udp "0.0.0.0:8090" +to_udp "127.0.0.1:514" ``` diff --git a/src/content/docs/reference/operators/from_udp.mdx b/src/content/docs/reference/operators/accept_udp.mdx similarity index 82% rename from src/content/docs/reference/operators/from_udp.mdx rename to src/content/docs/reference/operators/accept_udp.mdx index d55b4df41..1b8fcb176 100644 --- a/src/content/docs/reference/operators/from_udp.mdx +++ b/src/content/docs/reference/operators/accept_udp.mdx @@ -1,13 +1,13 @@ --- -title: from_udp +title: accept_udp category: Inputs/Events -example: 'from_udp "0.0.0.0:8090"' +example: 'accept_udp "0.0.0.0:8090"' --- Receives UDP datagrams and outputs structured events. ```tql -from_udp endpoint:string, [resolve_hostnames=bool], [binary=bool] +accept_udp endpoint:string, [resolve_hostnames=bool], [binary=bool] ``` ## Description @@ -15,8 +15,7 @@ from_udp endpoint:string, [resolve_hostnames=bool], [binary=bool] Listens for UDP datagrams on the specified endpoint and outputs each datagram as a structured event containing the data and peer information. -Unlike load_udp, which outputs raw bytes, -`from_udp` produces structured events with metadata about the sender. +`accept_udp` produces structured events with metadata about the sender. ### `endpoint: string` @@ -59,7 +58,7 @@ Each UDP datagram produces one event with the following structure: ### Receive UDP datagrams with sender information ```tql -from_udp "0.0.0.0:1234" +accept_udp "0.0.0.0:1234" ``` This might output events like: @@ -77,20 +76,19 @@ This might output events like: ### Parse JSON data from UDP datagrams ```tql -from_udp "127.0.0.1:8080" +accept_udp "127.0.0.1:8080" select data = data.parse_json() ``` ### Filter by sender and decode data ```tql -from_udp "0.0.0.0:9999" +accept_udp "0.0.0.0:9999" where peer.ip == 192.168.1.100 select data ``` ## See Also -- load_udp -- save_udp +- to_udp - udp diff --git a/src/content/docs/reference/operators/from.mdx b/src/content/docs/reference/operators/from.mdx index 47639293e..27de32d30 100644 --- a/src/content/docs/reference/operators/from.mdx +++ b/src/content/docs/reference/operators/from.mdx @@ -150,7 +150,6 @@ load_tcp "tcp://0.0.0.0:12345", parallel=10 { | `s3` | load_s3 | `from "s3://bucket/file.json"` | | `sqs` | load_sqs | `from "sqs://my-queue" { read_json }` | | `tcp` | load_tcp | `from "tcp://127.0.0.1:13245" { read_json }` | -| `udp` | load_udp | `from "udp://127.0.0.1:56789" { read_json }` | | `zmq` | load_zmq | `from "zmq://127.0.0.1:56789" { read_json }` | Please see the respective operator pages for details on the URI's locator format. diff --git a/src/content/docs/reference/operators/load_udp.mdx b/src/content/docs/reference/operators/load_udp.mdx deleted file mode 100644 index 2b296e1f1..000000000 --- a/src/content/docs/reference/operators/load_udp.mdx +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: load_udp -category: Inputs/Bytes -example: 'load_udp "0.0.0.0:8090"' ---- - -Loads bytes from a UDP socket. - -```tql -load_udp endpoint:str, [connect=bool, insert_newlines=bool] -``` - -## Description - -Loads bytes from a UDP socket. The operator defaults to creating a socket -in listening mode. Use `connect=true` if the operator should initiate the -connection instead. - -When you have a socket in listening mode, use `0.0.0.0` to accept connections -on all interfaces. The nics operator lists all all -available interfaces. - -### `endpoint: str` - -The address of the remote endpoint to load bytes from. Must be of the format: -`[udp://]host:port`. - -### `connect = bool (optional)` - -Connect to `endpoint` instead of listening at it. - -Defaults to `false`. - -### `insert_newlines = bool (optional)` - -Append a newline character (`\n`) at the end of every datagram. - -This option comes in handy in combination with line-based parsers downstream, -such as NDJSON. - -Defaults to `false`. - -## Examples - -### Import JSON via UDP by listenting on localhost - -```tql -load_udp "127.0.0.1:56789" -import -``` - -## See Also - -- save_udp -- udp diff --git a/src/content/docs/reference/operators/save_udp.mdx b/src/content/docs/reference/operators/save_udp.mdx deleted file mode 100644 index 5b2e6602f..000000000 --- a/src/content/docs/reference/operators/save_udp.mdx +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: save_udp -category: Outputs/Bytes -example: 'save_udp "0.0.0.0:8090"' ---- - -Saves bytes to a UDP socket. - -```tql -save_udp endpoint:str -``` - -## Description - -Saves bytes to a UDP socket. - -### `endpoint: str` - -The address of the remote endpoint to load bytes from. Must be of the format: -`[udp://]host:port`. - -## Examples - -Send the Tenzir version as CSV file to a remote endpoint via UDP: - -```tql -version -write_csv -save_udp "127.0.0.1:56789" -``` - -Use `nc -ul 127.0.0.1 56789` to spin up a UDP server to test the above pipeline. - -## See Also - -- load_udp -- udp diff --git a/src/content/docs/reference/operators/to.mdx b/src/content/docs/reference/operators/to.mdx index 56da630a5..a62f565e4 100644 --- a/src/content/docs/reference/operators/to.mdx +++ b/src/content/docs/reference/operators/to.mdx @@ -97,7 +97,6 @@ If no scheme is present, the connector attempts to save to the local filesystem. | `s3` | save_s3 | `to "s3://bucket/file.json"` | | `sqs` | save_sqs | `to "sqs://my-queue" { write_json }` | | `tcp` | save_tcp | `to "tcp://127.0.0.1:56789" { write_json }` | -| `udp` | save_udp | `to "udp://127.0.0.1:56789" { write_json }` | | `zmq` | save_zmq | `to "zmq://127.0.0.1:56789" { write_json }` | | `smtp`, `smtps`, `mailto`, `email` | save_email | `to "smtp://john@example.com"` | diff --git a/src/content/docs/reference/operators/to_udp.mdx b/src/content/docs/reference/operators/to_udp.mdx new file mode 100644 index 000000000..72ee719b3 --- /dev/null +++ b/src/content/docs/reference/operators/to_udp.mdx @@ -0,0 +1,58 @@ +--- +title: to_udp +category: Outputs/Events +example: 'to_udp "127.0.0.1:514"' +--- + +import Integration from '@components/see-also/Integration.astro'; +import Op from '@components/see-also/Op.astro'; + +Sends one UDP datagram per input event. + +```tql +to_udp endpoint:string, [message=string|blob|record] +``` + +## Description + +Sends one UDP datagram per input event to a fixed remote UDP endpoint. + +### `endpoint: string` + +The address of the remote endpoint. Must be of the format `host:port`. The +optional `udp://` prefix is accepted but not required. + +### `message = string|blob|record (optional)` + +An expression that is evaluated once per input event and must produce one of +the following types: + +- `string` values are sent as UTF-8 bytes. +- `blob` values are sent verbatim. +- `record` values are serialized as compact JSON. +- `null` values are skipped with a warning. + +All other result types are ignored with a warning. + +If you omit `message`, the operator sends the whole event as compact JSON. + +## Examples + +Send a field as a UDP datagram: + +```tql +from {message: "hello"} +to_udp "127.0.0.1:514", message=message +``` + +Send the full event as compact JSON: + +```tql +from {service: "dns", status: "ok"} +to_udp "127.0.0.1:514" +``` + +## See Also + +- accept_udp +- udp diff --git a/src/content/docs/reference/programs.mdx b/src/content/docs/reference/programs.mdx index 934d7da08..55e42e8b3 100644 --- a/src/content/docs/reference/programs.mdx +++ b/src/content/docs/reference/programs.mdx @@ -143,7 +143,7 @@ from_file "data.json" // Automatically uses read_json Operators that produce events directly, like from_kafka or -from_udp, don't take a parsing subpipeline +accept_udp, don't take a parsing subpipeline because the data format is inherent to the source. ## Comments