perf: fn rav1d_create_lf_mask_{intra,inter}: store as u16s

src/align.rs

@@ -263,6 +263,7 @@ impl<T: Copy, C: AlignedByteChunk> Default for AlignedVec<T, C> {
     }
 }
 
+pub type AlignedVec2<T> = AlignedVec<T, Align2<[u8; 2]>>;
 pub type AlignedVec32<T> = AlignedVec<T, Align32<[u8; 32]>>;
 pub type AlignedVec64<T> = AlignedVec<T, Align64<[u8; 64]>>;
 

src/decode.rs

@@ -4538,8 +4538,7 @@ pub(crate) fn rav1d_decode_frame_init(c: &Rav1dContext, fc: &Rav1dFrameContext)
     f.lf.mask.resize_with(num_sb128 as usize, Default::default);
     // over-allocate by 3 bytes since some of the SIMD implementations
     // index this from the level type and can thus over-read by up to 3 bytes.
-    f.lf.level
-        .resize_with(4 * num_sb128 as usize * 32 * 32 + 3, Default::default); // TODO: Fallible allocation
+    f.lf.level.resize(4 * num_sb128 as usize * 32 * 32 + 3, 0); // TODO: Fallible allocation
     if c.fc.len() > 1 {
         // TODO: Fallible allocation
         f.frame_thread

src/internal.rs

@@ -1,5 +1,6 @@
 use crate::align::Align16;
 use crate::align::Align64;
+use crate::align::AlignedVec2;
 use crate::align::AlignedVec64;
 use crate::cdef::Rav1dCdefDSPContext;
 use crate::cdf::CdfContext;
@@ -701,7 +702,7 @@ impl TxLpfRightEdge {
 #[derive(Default)]
 #[repr(C)]
 pub struct Rav1dFrameContextLf {
-    pub level: DisjointMut<Vec<u8>>,
+    pub level: DisjointMut<AlignedVec2<u8>>,
     pub mask: Vec<Av1Filter>, /* len = w*h */
     pub lr_mask: Vec<Av1Restoration>,
     pub lim_lut: Align16<Av1FilterLUT>,

src/lf_apply.rs

@@ -1,5 +1,6 @@
 #![deny(unsafe_op_in_unsafe_fn)]
 
+use crate::align::AlignedVec2;
 use crate::align::AlignedVec64;
 use crate::disjoint_mut::DisjointMut;
 use crate::include::common::bitdepth::BitDepth;
@@ -368,7 +369,7 @@ pub(crate) fn rav1d_copy_lpf<BD: BitDepth>(
 fn filter_plane_cols_y<BD: BitDepth>(
     f: &Rav1dFrameData,
     have_left: bool,
-    lvl: WithOffset<&DisjointMut<Vec<u8>>>,
+    lvl: WithOffset<&DisjointMut<AlignedVec2<u8>>>,
     mask: &[[[RelaxedAtomic<u16>; 2]; 3]; 32],
     y_dst: Rav1dPictureDataComponentOffset,
     w: usize,
@@ -405,7 +406,7 @@ fn filter_plane_cols_y<BD: BitDepth>(
 fn filter_plane_rows_y<BD: BitDepth>(
     f: &Rav1dFrameData,
     have_top: bool,
-    lvl: WithOffset<&DisjointMut<Vec<u8>>>,
+    lvl: WithOffset<&DisjointMut<AlignedVec2<u8>>>,
     b4_stride: usize,
     mask: &[[[RelaxedAtomic<u16>; 2]; 3]; 32],
     y_dst: Rav1dPictureDataComponentOffset,
@@ -437,7 +438,7 @@ fn filter_plane_rows_y<BD: BitDepth>(
 fn filter_plane_cols_uv<BD: BitDepth>(
     f: &Rav1dFrameData,
     have_left: bool,
-    lvl: WithOffset<&DisjointMut<Vec<u8>>>,
+    lvl: WithOffset<&DisjointMut<AlignedVec2<u8>>>,
     mask: &[[[RelaxedAtomic<u16>; 2]; 2]; 32],
     u_dst: Rav1dPictureDataComponentOffset,
     v_dst: Rav1dPictureDataComponentOffset,
@@ -480,7 +481,7 @@ fn filter_plane_cols_uv<BD: BitDepth>(
 fn filter_plane_rows_uv<BD: BitDepth>(
     f: &Rav1dFrameData,
     have_top: bool,
-    lvl: WithOffset<&DisjointMut<Vec<u8>>>,
+    lvl: WithOffset<&DisjointMut<AlignedVec2<u8>>>,
     b4_stride: usize,
     mask: &[[[RelaxedAtomic<u16>; 2]; 2]; 32],
     u_dst: Rav1dPictureDataComponentOffset,

src/lf_mask.rs

@@ -1,4 +1,5 @@
 use crate::align::Align16;
+use crate::align::AlignedVec2;
 use crate::align::ArrayDefault;
 use crate::ctx::CaseSet;
 use crate::disjoint_mut::DisjointMut;
@@ -19,6 +20,7 @@ use parking_lot::RwLock;
 use std::cmp;
 use std::ffi::c_int;
 use std::mem::MaybeUninit;
+use zerocopy::FromBytes;
 
 #[repr(C)]
 pub struct Av1FilterLUT {
@@ -425,7 +427,7 @@ fn mask_edges_chroma(
 #[inline(never)]
 pub(crate) fn rav1d_create_lf_mask_intra(
     lflvl: &Av1Filter,
-    level_cache: &DisjointMut<Vec<u8>>,
+    level_cache: &DisjointMut<AlignedVec2<u8>>,
     b4_stride: ptrdiff_t,
     filter_level: &Align16<[[[u8; 2]; 8]; 4]>,
     b: Bxy,
@@ -449,15 +451,17 @@ pub(crate) fn rav1d_create_lf_mask_intra(
     let bx4 = bx & 31;
     let by4 = by & 31;
 
+    let filter_level_yuv = filter_level.0.map(|a| a[0][0]);
+    let [filter_level_y, filter_level_uv] = *<[u16; 2]>::ref_from(&filter_level_yuv).unwrap();
+
     if bw4 != 0 && bh4 != 0 {
         let mut level_cache_off = by * b4_stride + bx;
         for _y in 0..bh4 {
             for x in 0..bw4 {
                 let idx = 4 * (level_cache_off + x);
                 // `0.., ..2` is for Y
                 let lvl = &mut *level_cache.index_mut((idx + 0.., ..2));
-                lvl[0] = filter_level[0][0][0];
-                lvl[1] = filter_level[1][0][0];
+                *u16::mut_from(lvl).unwrap() = filter_level_y;
             }
             level_cache_off += b4_stride;
         }
@@ -494,8 +498,7 @@ pub(crate) fn rav1d_create_lf_mask_intra(
             let idx = 4 * (level_cache_off + x);
             // `2.., ..2` is for UV
             let lvl = &mut *level_cache.index_mut((idx + 2.., ..2));
-            lvl[0] = filter_level[2][0][0];
-            lvl[1] = filter_level[3][0][0];
+            *u16::mut_from(lvl).unwrap() = filter_level_uv;
         }
         level_cache_off += b4_stride;
     }
@@ -518,7 +521,7 @@ pub(crate) fn rav1d_create_lf_mask_intra(
 #[inline(never)]
 pub(crate) fn rav1d_create_lf_mask_inter(
     lflvl: &Av1Filter,
-    level_cache: &DisjointMut<Vec<u8>>,
+    level_cache: &DisjointMut<AlignedVec2<u8>>,
     b4_stride: ptrdiff_t,
     filter_level: &Align16<[[[u8; 2]; 8]; 4]>,
     r#ref: usize,
@@ -547,15 +550,17 @@ pub(crate) fn rav1d_create_lf_mask_inter(
     let bx4 = bx & 31;
     let by4 = by & 31;
 
+    let filter_level_yuv = filter_level.0.map(|a| a[r#ref][is_gmv]);
+    let [filter_level_y, filter_level_uv] = *<[u16; 2]>::ref_from(&filter_level_yuv).unwrap();
+
     if bw4 != 0 && bh4 != 0 {
         let mut level_cache_off = by * b4_stride + bx;
         for _y in 0..bh4 {
             for x in 0..bw4 {
                 let idx = 4 * (level_cache_off + x);
                 // `0.., ..2` is for Y
                 let lvl = &mut *level_cache.index_mut((idx + 0.., ..2));
-                lvl[0] = filter_level[0][r#ref][is_gmv];
-                lvl[1] = filter_level[1][r#ref][is_gmv];
+                *u16::mut_from(lvl).unwrap() = filter_level_y;
             }
             level_cache_off += b4_stride;
         }
@@ -603,8 +608,7 @@ pub(crate) fn rav1d_create_lf_mask_inter(
             let idx = 4 * (level_cache_off + x);
             // `2.., ..2` is for UV
             let lvl = &mut *level_cache.index_mut((idx + 2.., ..2));
-            lvl[0] = filter_level[2][r#ref][is_gmv];
-            lvl[1] = filter_level[3][r#ref][is_gmv];
+            *u16::mut_from(lvl).unwrap() = filter_level_uv;
         }
         level_cache_off += b4_stride;
     }

src/loopfilter.rs

@@ -1,6 +1,7 @@
 #![deny(unsafe_op_in_unsafe_fn)]
 
 use crate::align::Align16;
+use crate::align::AlignedVec2;
 use crate::cpu::CpuFlags;
 use crate::disjoint_mut::DisjointMut;
 use crate::ffi_safe::FFISafe;
@@ -34,7 +35,7 @@ wrap_fn_ptr!(pub unsafe extern "C" fn loopfilter_sb(
     w: c_int,
     bitdepth_max: c_int,
     _dst: *const FFISafe<Rav1dPictureDataComponentOffset>,
-    _lvl: *const FFISafe<WithOffset<&DisjointMut<Vec<u8>>>>,
+    _lvl: *const FFISafe<WithOffset<&DisjointMut<AlignedVec2<u8>>>>,
 ) -> ());
 
 impl loopfilter_sb::Fn {
@@ -43,7 +44,7 @@ impl loopfilter_sb::Fn {
         f: &Rav1dFrameData,
         dst: Rav1dPictureDataComponentOffset,
         mask: &[u32; 3],
-        lvl: WithOffset<&DisjointMut<Vec<u8>>>,
+        lvl: WithOffset<&DisjointMut<AlignedVec2<u8>>>,
         w: usize,
     ) {
         let dst_ptr = dst.as_mut_ptr::<BD>().cast();
@@ -289,7 +290,7 @@ enum YUV {
 fn loop_filter_sb128_rust<BD: BitDepth, const HV: usize, const YUV: usize>(
     mut dst: Rav1dPictureDataComponentOffset,
     vmask: &[u32; 3],
-    mut lvl: WithOffset<&DisjointMut<Vec<u8>>>,
+    mut lvl: WithOffset<&DisjointMut<AlignedVec2<u8>>>,
     b4_stride: usize,
     lut: &Align16<Av1FilterLUT>,
     _wh: c_int,
@@ -367,7 +368,7 @@ unsafe extern "C" fn loop_filter_sb128_c_erased<BD: BitDepth, const HV: usize, c
     wh: c_int,
     bitdepth_max: c_int,
     dst: *const FFISafe<Rav1dPictureDataComponentOffset>,
-    lvl: *const FFISafe<WithOffset<&DisjointMut<Vec<u8>>>>,
+    lvl: *const FFISafe<WithOffset<&DisjointMut<AlignedVec2<u8>>>>,
 ) {
     // SAFETY: Was passed as `FFISafe::new(_)` in `loopfilter_sb::Fn::call`.
     let dst = *unsafe { FFISafe::get(dst) };