diff --git a/src/content/docs/integrations/ftp.mdx b/src/content/docs/integrations/ftp.mdx index 74579f6a5..7842faaa7 100644 --- a/src/content/docs/integrations/ftp.mdx +++ b/src/content/docs/integrations/ftp.mdx @@ -4,7 +4,9 @@ title: FTP Tenzir supports the [File Transfer Protocol (FTP)](https://en.wikipedia.org/wiki/File_Transfer_Protocol), both downloading -and uploading files. +and uploading files. Use from_ftp to download bytes and parse them +with a subpipeline, and use to_ftp to print events with a subpipeline +and upload the result. ![FTP](ftp.svg) @@ -12,28 +14,33 @@ FTP consists of two separate TCP connections, one control and one data connection. This can be tricky for some firewalls and may require special attention. -:::tip[URL Support] -The URL schemes `ftp://` and `ftps://` dispatch to -load_ftp and -save_ftp for seamless URL-style use via -from and -to. -::: - ## Examples -### Download a file from an FTP server +These examples use the direct FTP operators with explicit parsing and printing +subpipelines. + +### Download and parse a file from an FTP server + +Use from_ftp with read_ndjson to turn the downloaded bytes +into events. ```tql -from "ftp://user:pass@ftp.example.org/path/to/file.json" +from_ftp "ftp://user:pass@ftp.example.org/path/to/file.ndjson" { + read_ndjson +} ``` ### Upload events to an FTP server +Use to_ftp with write_ndjson to serialize events before +uploading them. + ```tql from { x: 42, y: "foo", } -to "ftp://user:pass@ftp.example.org/a/b/c/events.json.gz" +to_ftp "ftp://user:pass@ftp.example.org/a/b/c/events.ndjson" { + write_ndjson +} ``` diff --git a/src/content/docs/reference/operators.mdx b/src/content/docs/reference/operators.mdx index 468b963f1..865189096 100644 --- a/src/content/docs/reference/operators.mdx +++ b/src/content/docs/reference/operators.mdx @@ -343,6 +343,10 @@ operators: description: 'Reads one or multiple files from a filesystem.' example: 'from_file "s3://data/**.json"' path: 'reference/operators/from_file' + - name: 'from_ftp' + description: 'Downloads bytes via FTP and parses them with a subpipeline.' + example: 'from_ftp "ftp.example.org/events.ndjson" { read_ndjson }' + path: 'reference/operators/from_ftp' - name: 'from_fluent_bit' description: 'Receives events via Fluent Bit.' example: 'from_fluent_bit "opentelemetry"' @@ -395,10 +399,6 @@ operators: description: 'Loads the contents of the file at `path` as a byte stream.' example: 'load_file "/tmp/data.json"' path: 'reference/operators/load_file' - - name: 'load_ftp' - description: 'Loads a byte stream via FTP.' - example: 'load_ftp "ftp.example.org"' - path: 'reference/operators/load_ftp' - name: 'load_gcs' description: 'Loads bytes from a Google Cloud Storage object.' example: 'load_gcs "gs://bucket/object.json"' @@ -635,10 +635,6 @@ operators: description: 'Writes a byte stream to a file.' example: 'save_file "/tmp/out.json"' path: 'reference/operators/save_file' - - name: 'save_ftp' - description: 'Saves a byte stream via FTP.' - example: 'save_ftp "ftp.example.org"' - path: 'reference/operators/save_ftp' - name: 'save_gcs' description: 'Saves bytes to a Google Cloud Storage object.' example: 'save_gcs "gs://bucket/object.json"' @@ -707,6 +703,10 @@ operators: description: 'Sends events via Fluent Bit.' example: 'to_fluent_bit "elasticsearch" …' path: 'reference/operators/to_fluent_bit' + - name: 'to_ftp' + description: 'Prints events to bytes and uploads them via FTP.' + example: 'to_ftp "ftp.example.org/events.ndjson" { write_ndjson }' + path: 'reference/operators/to_ftp' - name: 'to_google_cloud_logging' description: 'Sends events to Google Cloud Logging.' example: 'to_google_cloud_logging …' @@ -2018,14 +2018,6 @@ load_file "/tmp/data.json" - - -```tql -load_ftp "ftp.example.org" -``` - - - ```tql @@ -2143,6 +2135,14 @@ from_file "s3://data/**.json" + + +```tql +from_ftp "ftp.example.org/events.ndjson" { read_ndjson } +``` + + + ```tql @@ -2356,14 +2356,6 @@ save_file "/tmp/out.json" - - -```tql -save_ftp "ftp.example.org" -``` - - - ```tql @@ -2489,6 +2481,14 @@ to_fluent_bit "elasticsearch" … + + +```tql +to_ftp "ftp.example.org/events.ndjson" { write_ndjson } +``` + + + ```tql diff --git a/src/content/docs/reference/operators/from.mdx b/src/content/docs/reference/operators/from.mdx index f077c073a..e4d6e3a94 100644 --- a/src/content/docs/reference/operators/from.mdx +++ b/src/content/docs/reference/operators/from.mdx @@ -141,7 +141,6 @@ load_tcp "tcp://0.0.0.0:12345", parallel=10 { | `elasticsearch` | from_opensearch | `from "elasticsearch://1.2.3.4:9200` | | `file` | load_file | `from "file://path/to/file.json"` | | `fluent-bit` | from_fluent_bit | `from "fluent-bit://elasticsearch"` | -| `ftp`, `ftps` | load_ftp | `from "ftp://example.com/file.json"` | | `gs` | load_gcs | `from "gs://bucket/object.json"` | | `http`, `https` | load_http | `from "http://example.com/file.json"` | | `inproc` | load_zmq | `from "inproc://127.0.0.1:56789" { read_json }` | diff --git a/src/content/docs/reference/operators/from_ftp.mdx b/src/content/docs/reference/operators/from_ftp.mdx new file mode 100644 index 000000000..584a29feb --- /dev/null +++ b/src/content/docs/reference/operators/from_ftp.mdx @@ -0,0 +1,68 @@ +--- +title: from_ftp +category: Inputs/Events +example: 'from_ftp "ftp.example.org/events.ndjson" { read_ndjson }' +--- + +Downloads bytes via FTP or FTPS and parses them with a subpipeline. + +```tql +from_ftp url:string, [tls=record] { … } +``` + +## Description + +The `from_ftp` operator downloads bytes from an FTP or FTPS server and forwards +them to the required subpipeline. + +### `url: string` + +The URL to request from. You can omit the `ftp://` scheme. + +### `tls = record (optional)` + +TLS configuration. + +By default, `ftps://` enables TLS and `ftp://` does not. If you omit the +scheme, the operator assumes `ftp://`. + +import TLSOptions from '@partials/operators/TLSOptions.mdx'; + + + +### `{ … }` + +A required parsing subpipeline. + +The subpipeline receives the downloaded body as bytes and must return events. +For example, use read_ndjson to parse newline-delimited JSON. + +## Examples + +### Download NDJSON from an FTP server + +Use read_ndjson when the remote file already contains +newline-delimited JSON. + +```tql +from_ftp "ftp://user:pass@ftp.example.org/events.ndjson" { + read_ndjson +} +``` + +### Download gzipped JSON and decompress it explicitly + +Decompress the downloaded bytes before parsing them when the remote file is +stored as gzip-compressed JSON. + +```tql +from_ftp "ftp://user:pass@ftp.example.org/events.json.gz" { + decompress gzip + read_json +} +``` + +## See Also + +- to_ftp +- ftp diff --git a/src/content/docs/reference/operators/load_ftp.mdx b/src/content/docs/reference/operators/load_ftp.mdx deleted file mode 100644 index f075147fb..000000000 --- a/src/content/docs/reference/operators/load_ftp.mdx +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: load_ftp -category: Inputs/Bytes -example: 'load_ftp "ftp.example.org"' ---- - -Loads a byte stream via FTP. - -```tql -load_ftp url:str, [tls=record] -``` - -## Description - -Loads a byte stream via FTP. - -### `url: str` - -The URL to request from. The `ftp://` scheme can be omitted. - -import TLSOptions from '@partials/operators/TLSOptions.mdx'; - - - -## Examples - -```tql -load_ftp "ftp.example.org" -``` - -## See Also - -- save_ftp -- ftp diff --git a/src/content/docs/reference/operators/save_ftp.mdx b/src/content/docs/reference/operators/save_ftp.mdx deleted file mode 100644 index df6fa672b..000000000 --- a/src/content/docs/reference/operators/save_ftp.mdx +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: save_ftp -category: Outputs/Bytes -example: 'save_ftp "ftp.example.org"' ---- - -Saves a byte stream via FTP. - -```tql -save_ftp url:str [tls=record] -``` - -## Description - -Saves a byte stream via FTP. - -### `url: str` - -The URL to request from. The `ftp://` scheme can be omitted. - -import TLSOptions from '@partials/operators/TLSOptions.mdx'; - - - -## Examples - -```tql -save_ftp "ftp.example.org" -``` - -## See Also - -- load_ftp -- ftp diff --git a/src/content/docs/reference/operators/to.mdx b/src/content/docs/reference/operators/to.mdx index 56da630a5..4cb701d3d 100644 --- a/src/content/docs/reference/operators/to.mdx +++ b/src/content/docs/reference/operators/to.mdx @@ -88,7 +88,6 @@ If no scheme is present, the connector attempts to save to the local filesystem. | `elasticsearch` | to_opensearch | `to "elasticsearch://…` | | `file` | save_file | `to "file://path/to/file.json"` | | `fluent-bit` | to_fluent_bit | `to "fluent-bit://elasticsearch"` | -| `ftp`, `ftps` | save_ftp | `to "ftp://example.com/file.json"` | | `gs` | save_gcs | `to "gs://bucket/object.json"` | | `http`, `https` | save_http | `to "http://example.com/file.json"` | | `inproc` | save_zmq | `to "inproc://127.0.0.1:56789" { write_json }` | diff --git a/src/content/docs/reference/operators/to_ftp.mdx b/src/content/docs/reference/operators/to_ftp.mdx new file mode 100644 index 000000000..675694efa --- /dev/null +++ b/src/content/docs/reference/operators/to_ftp.mdx @@ -0,0 +1,77 @@ +--- +title: to_ftp +category: Outputs/Events +example: 'to_ftp "ftp.example.org/events.ndjson" { write_ndjson }' +--- + +Prints events to bytes and uploads them via FTP or FTPS. + +```tql +to_ftp url:string, [tls=record] { … } +``` + +## Description + +The `to_ftp` operator sends events to an FTP or FTPS server. + +The required subpipeline receives events and must return bytes. + +### `url: string` + +The URL to upload to. You can omit the `ftp://` scheme. + +### `tls = record (optional)` + +TLS configuration. + +By default, `ftps://` enables TLS and `ftp://` does not. If you omit the +scheme, the operator assumes `ftp://`. + +import TLSOptions from '@partials/operators/TLSOptions.mdx'; + + + +### `{ … }` + +A required printing subpipeline. + +The subpipeline receives events and must return bytes. For example, use +write_ndjson to serialize events as newline-delimited JSON. + +## Examples + +### Upload events as NDJSON + +Use write_ndjson to serialize each event as one JSON object per line +before uploading it. + +```tql +from { + x: 42, + y: "foo", +} +to_ftp "ftp://user:pass@ftp.example.org/events.ndjson" { + write_ndjson +} +``` + +### Upload compressed NDJSON + +Add compress gzip to the printing subpipeline when you want to upload +compressed output. + +```tql +from { + x: 42, + y: "foo", +} +to_ftp "ftp://user:pass@ftp.example.org/events.ndjson.gz" { + write_ndjson + compress gzip +} +``` + +## See Also + +- from_ftp +- ftp