trim

.github/workflows/main.yml

@@ -103,3 +103,18 @@ jobs:
     - name: Install Rust
       run: rustup update stable && rustup default stable && rustup component add rustfmt
     - run: cargo fmt -- --check
+
+  upstream-c-tests:
+    name: Upstream C xz tests with xz-sys
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - name: Install Rust
+      run: rustup update stable --no-self-update && rustup default stable
+    - uses: Swatinem/rust-cache@v2
+      with:
+        cache-targets: "false"
+    - name: Run upstream C xz test harness against xz-sys
+      run: scripts/run_xz_c_tests_with_xz_sys.sh

README.md

@@ -10,21 +10,21 @@
 
 Pure Rust xz2/liblzma-compatible crates for reading and writing xz streams.
 
-**This crate is forked from [xz2](https://crates.io/crates/xz2) and `xz = "0.1.x"` is fully compatible with `xz2 = "0.1.7"`,**
+**This crate is forked from [liblzma](https://crates.io/crates/liblzma) and `xz = "0.4.x"` is fully compatible with `liblzma = "0.4.6"`,**
 so you can migrate simply.
 
 ## Migrate from xz2
 
 ```diff
 # Cargo.toml
 [dependencies]
--xz2 = "0.1.7"
-+xz = "0.1.7"
+-liblzma = "0.4.6"
++xz = "0.4.6"
 ```
 
 ```diff
 // *.rs
--use xz2;
+-use liblzma;
 +use xz;
 ```
 

scripts/run_xz_c_tests_with_xz_sys.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+XZ_SRC_DIR="$ROOT_DIR/liblzma-sys/xz"
+BUILD_DIR="${XZ_SYS_C_TEST_BUILD_DIR:-$ROOT_DIR/target/xz-sys-c-tests}"
+CARGO_BUILD_DIR="$BUILD_DIR/cargo"
+CMAKE_BUILD_DIR="$BUILD_DIR/cmake"
+JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)}"
+
+if command -v ninja >/dev/null 2>&1; then
+    CMAKE_GENERATOR=( -G Ninja )
+else
+    CMAKE_GENERATOR=()
+fi
+
+case "$(uname -s)" in
+    Linux)
+        # Rust staticlibs require these native libraries when linked by CMake.
+        CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS:-} -pthread -ldl -lm -lrt"
+        ;;
+    *)
+        CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS:-}"
+        ;;
+esac
+
+echo "Building xz-sys staticlib..."
+CARGO_TARGET_DIR="$CARGO_BUILD_DIR" cargo build -p xz-sys --release --features parallel
+
+RUST_LIB="$CARGO_BUILD_DIR/release/libxz_sys.a"
+CMAKE_LIB="$CMAKE_BUILD_DIR/liblzma.a"
+
+replace_liblzma_with_xz_sys() {
+    cp "$RUST_LIB" "$CMAKE_LIB"
+
+    ar t "$CMAKE_LIB" > "$BUILD_DIR/liblzma-archive-members.txt"
+    if ! grep -q 'xz_sys' "$BUILD_DIR/liblzma-archive-members.txt"; then
+        echo "replacement liblzma archive does not look like xz-sys" >&2
+        exit 1
+    fi
+}
+
+echo "Configuring upstream xz C test harness..."
+cmake -S "$XZ_SRC_DIR" -B "$CMAKE_BUILD_DIR" "${CMAKE_GENERATOR[@]}" \
+    -DBUILD_SHARED_LIBS=OFF \
+    -DBUILD_TESTING=ON \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_EXE_LINKER_FLAGS="$CMAKE_EXE_LINKER_FLAGS" \
+    -DXZ_NLS=OFF \
+    -DXZ_DOXYGEN=OFF
+
+echo "Building C liblzma target once to materialize CMake outputs..."
+cmake --build "$CMAKE_BUILD_DIR" --target liblzma --parallel "$JOBS"
+
+echo "Replacing C liblzma with xz-sys and building all C tests/tools..."
+replace_liblzma_with_xz_sys
+cmake --build "$CMAKE_BUILD_DIR" --parallel "$JOBS"
+
+# Replace once more and rebuild dependents. This guarantees that CMake test
+# executables and tools are linked against xz-sys even if the previous build
+# regenerated liblzma.a as part of the all target.
+replace_liblzma_with_xz_sys
+cmake --build "$CMAKE_BUILD_DIR" --parallel "$JOBS"
+
+echo "Running upstream xz C tests against xz-sys..."
+ctest --test-dir "$CMAKE_BUILD_DIR" --output-on-failure "$@"

xz-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xz-core"
-version = "0.1.0"
+version = "0.0.1"
 edition = "2024"
 license = "MIT OR Apache-2.0"
 description = "Pure Rust implementation of liblzma (transpiled from C via c2rust)"

xz-core/src/alloc.rs

@@ -17,9 +17,12 @@ pub use custom::allocator_or_c;
 #[cfg(feature = "custom_allocator")]
 pub use rust::rust_allocator;
 
+#[cfg(feature = "custom_allocator")]
 pub use policy::{lzma_alloc, lzma_alloc_zero, lzma_free};
 
 pub(crate) use policy::{
-    internal_alloc_array, internal_alloc_bytes, internal_alloc_object, internal_alloc_zeroed_array,
-    internal_free, internal_free_array, internal_free_bytes,
+    internal_alloc_array, internal_alloc_bytes, internal_alloc_object,
+    internal_alloc_untyped_bytes, internal_alloc_zeroed_array, internal_alloc_zeroed_bytes,
+    internal_free, internal_free_array, internal_free_bytes, internal_free_untyped,
+    internal_free_untyped_bytes,
 };

xz-core/src/alloc/custom.rs

@@ -70,6 +70,29 @@ pub(crate) unsafe fn internal_alloc_bytes(
     alloc_impl(size as usize, RUST_ALLOC_ALIGN, false)
 }
 
+pub(crate) unsafe fn internal_alloc_untyped_bytes(
+    size: size_t,
+    allocator: *const lzma_allocator,
+) -> *mut c_void {
+    unsafe { internal_alloc_bytes(size, allocator) }
+}
+
+pub(crate) unsafe fn internal_alloc_zeroed_bytes(
+    size: size_t,
+    allocator: *const lzma_allocator,
+) -> *mut c_void {
+    let size = c_size(size);
+    let allocator = allocator_or_rust(allocator);
+    if let Some(alloc) = unsafe { (*allocator).alloc } {
+        let ptr = unsafe { alloc((*allocator).opaque, 1, size) };
+        if !ptr.is_null() {
+            unsafe { core::ptr::write_bytes(ptr.cast::<u8>(), 0, size) };
+        }
+        return ptr;
+    }
+    alloc_impl(size as usize, RUST_ALLOC_ALIGN, true)
+}
+
 pub(crate) unsafe fn internal_alloc_object<T>(allocator: *const lzma_allocator) -> *mut T {
     if !allocator.is_null()
         && let Some(alloc) = unsafe { (*allocator).alloc }
@@ -115,7 +138,11 @@ pub(crate) unsafe fn internal_alloc_zeroed_array<T>(
     alloc_impl(size, core::mem::align_of::<T>(), true) as *mut T
 }
 
-pub(crate) unsafe fn internal_free_bytes(ptr: *mut c_void, allocator: *const lzma_allocator) {
+pub(crate) unsafe fn internal_free_bytes(
+    ptr: *mut c_void,
+    _size: size_t,
+    allocator: *const lzma_allocator,
+) {
     if !allocator.is_null()
         && let Some(free) = unsafe { (*allocator).free }
     {
@@ -125,14 +152,25 @@ pub(crate) unsafe fn internal_free_bytes(ptr: *mut c_void, allocator: *const lzm
     unsafe { free_ptr(ptr) };
 }
 
+pub(crate) unsafe fn internal_free_untyped(ptr: *mut c_void, allocator: *const lzma_allocator) {
+    unsafe { lzma_free(ptr, allocator) };
+}
+
+pub(crate) unsafe fn internal_free_untyped_bytes(
+    ptr: *mut c_void,
+    allocator: *const lzma_allocator,
+) {
+    unsafe { lzma_free(ptr, allocator) };
+}
+
 pub(crate) unsafe fn internal_free<T>(ptr: *mut T, allocator: *const lzma_allocator) {
-    unsafe { internal_free_bytes(ptr.cast(), allocator) };
+    unsafe { internal_free_bytes(ptr.cast(), 0, allocator) };
 }
 
 pub(crate) unsafe fn internal_free_array<T>(
     ptr: *mut T,
     _count: size_t,
     allocator: *const lzma_allocator,
 ) {
-    unsafe { internal_free_bytes(ptr.cast(), allocator) };
+    unsafe { internal_free_bytes(ptr.cast(), 0, allocator) };
 }

xz-core/src/common/common.rs

@@ -1,3 +1,4 @@
+#[cfg(feature = "custom_allocator")]
 pub use crate::alloc::{lzma_alloc, lzma_alloc_zero, lzma_free};
 use crate::types::*;
 pub const LZMA_VERSION_MAJOR: u32 = 5;
@@ -31,7 +32,7 @@ pub unsafe fn lzma_stream_allocator(strm: *const lzma_stream) -> *const lzma_all
 #[inline]
 pub unsafe fn lzma_alloc_object<T>(allocator: *const lzma_allocator) -> *mut T {
     debug_assert!(core::mem::align_of::<T>() <= 16);
-    lzma_alloc(core::mem::size_of::<T>() as size_t, allocator) as *mut T
+    crate::alloc::internal_alloc_object::<T>(allocator)
 }
 pub unsafe fn lzma_bufcpy(
     input: *const u8,
@@ -113,7 +114,7 @@ pub unsafe fn lzma_next_end(next: *mut lzma_next_coder, allocator: *const lzma_a
     if let Some(end) = (*next).end {
         end((*next).coder, allocator);
     } else {
-        lzma_free((*next).coder, allocator);
+        crate::alloc::internal_free_untyped((*next).coder, allocator);
     }
     *next = lzma_next_coder_s {
         coder: core::ptr::null_mut(),
@@ -297,7 +298,7 @@ pub unsafe fn lzma_end(strm: *mut lzma_stream) {
             ::core::ptr::addr_of_mut!((*(*strm).internal).next),
             lzma_stream_allocator(strm),
         );
-        lzma_free((*strm).internal as *mut c_void, lzma_stream_allocator(strm));
+        crate::alloc::internal_free((*strm).internal, lzma_stream_allocator(strm));
         (*strm).internal = core::ptr::null_mut();
     }
 }

xz-core/src/common/filter_common.rs

@@ -101,6 +101,37 @@ static FEATURES: [filter_features; 13] = [
         changes_size: false,
     },
 ];
+
+fn filter_options_size(id: lzma_vli) -> Option<size_t> {
+    let mut i: usize = 0;
+    while i < FEATURES.len() {
+        if FEATURES[i].id == id {
+            return Some(FEATURES[i].options_size);
+        }
+        if FEATURES[i].id == LZMA_VLI_UNKNOWN {
+            break;
+        }
+        i += 1;
+    }
+    None
+}
+
+pub(crate) unsafe fn lzma_filter_options_free(
+    filter: lzma_filter,
+    allocator: *const lzma_allocator,
+) {
+    if filter.options.is_null() {
+        return;
+    }
+    if let Some(size) = filter_options_size(filter.id) {
+        crate::alloc::internal_free_bytes(filter.options, size, allocator);
+    } else {
+        debug_assert!(false, "unknown filter options size");
+        #[cfg(feature = "custom_allocator")]
+        lzma_free(filter.options, allocator);
+    }
+}
+
 pub unsafe fn lzma_filters_copy(
     src: *const lzma_filter,
     real_dest: *mut lzma_filter,
@@ -138,7 +169,10 @@ pub unsafe fn lzma_filters_copy(
                         j += 1;
                     }
                 }
-                dest[i as usize].options = lzma_alloc(FEATURES[j as usize].options_size, allocator);
+                dest[i as usize].options = crate::alloc::internal_alloc_bytes(
+                    FEATURES[j as usize].options_size,
+                    allocator,
+                );
                 if dest[i as usize].options.is_null() {
                     ret = LZMA_MEM_ERROR;
                     break 's_15;
@@ -156,7 +190,7 @@ pub unsafe fn lzma_filters_copy(
     if ret != LZMA_OK {
         while i > 0 {
             i -= 1;
-            lzma_free(dest[i as usize].options, allocator);
+            lzma_filter_options_free(dest[i as usize], allocator);
         }
         return ret;
     }
@@ -178,7 +212,7 @@ pub unsafe fn lzma_filters_free(filters: *mut lzma_filter, allocator: *const lzm
         if i == LZMA_FILTERS_MAX as size_t {
             break;
         }
-        lzma_free((*filters.offset(i as isize)).options, allocator);
+        lzma_filter_options_free(*filters.offset(i as isize), allocator);
         (*filters.offset(i as isize)).options = core::ptr::null_mut();
         (*filters.offset(i as isize)).id = LZMA_VLI_UNKNOWN;
         i += 1;

xz-core/src/common/filter_encoder.rs

@@ -261,7 +261,7 @@ unsafe fn reversed_filter_slot(filters: *mut [lzma_filter; 5], index: usize) ->
     (filters as *mut lzma_filter).add(index)
 }
 #[inline(always)]
-unsafe fn supported_action_slot(actions: *mut bool, index: u32) -> *mut bool {
+unsafe fn supported_action_slot(actions: *mut bool, index: lzma_action) -> *mut bool {
     debug_assert!((index as usize) < 5);
     actions.add(index as usize)
 }