perf(worker-core): replace AsyncArrowWriter with parquet-ext

Cargo.toml

@@ -165,6 +165,7 @@ datafusion = { version = "52", features = ["serde"] }
 datafusion-tracing = { version = "52" }
 datafusion-datasource = { version = "52" }
 object_store = { version = "0.12", features = ["aws", "gcp", "azure"] }
+parquet-ext = { version = "0.3.0", git = "https://github.com/edgeandnode/parquet-ext.git", rev = "66855214cd1969911e459e78242c36eb91be3951", features = ["zstd"] }
 
 # Crates that should follow the version used by DataFusion and Arrow
 prost = "0.14.1"

crates/core/worker-core/Cargo.toml

@@ -25,3 +25,4 @@ thiserror.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 url.workspace = true
+parquet-ext.workspace = true

crates/core/worker-core/src/parquet_writer.rs

@@ -12,15 +12,16 @@
         },
     },
     parquet::{
-        arrow::AsyncArrowWriter,
         errors::ParquetError,
         file::{metadata::KeyValue, properties::WriterProperties as ParquetWriterProperties},
     },
 };
+use parquet_ext::AsyncArrowWriter;
+
 use metadata_db::{
     MetadataDb,
     files::{FileId, FooterBytes},
     physical_table_revision::LocationId,
 };
 use object_store::{ObjectMeta, buffered::BufWriter};
 use url::Url;