[WIP] Switch back to arrow-rs

Cargo.toml

@@ -10,7 +10,9 @@ exclude = ["benches", "tests"]
 readme = "README.md"
 
 [dependencies]
-arrow2 = { version = "0.17", features = ["io_ipc", "io_ipc_compression", "io_json" ] }
+arrow = "52"
+arrow-buffer = "52"
+arrow-ipc = "52"
 byteorder = "1"
 encoding_rs = "0.8"
 log = "0.4"
@@ -22,7 +24,7 @@ thiserror = "1.0"
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 
 [dev-dependencies]
-arrow2 = { version = "0.17", features = ["io_json"] }
+arrow-json = "52"
 criterion = "0.5"
 iai-callgrind = "0.11"
 pretty_assertions = "1.3"

gen/resources/preamble/immutable/mod.rs

@@ -4,7 +4,7 @@
 //! [`crate::io::peppi::read`].
 //!
 //! These arrays can be shared, and cloning them is `O(1)`. See the
-//! [arrow2 docs](https://docs.rs/arrow2/latest/arrow2/array/index.html) for more.
+//! [arrow_array docs](https://docs.rs/arrow-array/latest/arrow_array/index.html) for more.
 
 #![allow(unused_variables)]
 
@@ -13,16 +13,17 @@ mod slippi;
 
 use std::fmt;
 
-use arrow2::{
-	array::PrimitiveArray,
-	bitmap::Bitmap,
-	buffer::Buffer,
-	offset::OffsetsBuffer,
+use arrow::{
+	array::{
+		types::{Float32Type, Int8Type, Int32Type, UInt8Type, UInt16Type, UInt32Type},
+		PrimitiveArray,
+	},
+	buffer::{NullBuffer, OffsetBuffer},
 };
 
 use crate::{
 	io::slippi::Version,
-	frame::{self, mutable, transpose, Rollbacks},
+	frame::{self, transpose, Rollbacks},
 	game::Port,
 };
 
@@ -31,7 +32,7 @@ use crate::{
 pub struct Data {
 	pub pre: Pre,
 	pub post: Post,
-	pub validity: Option<Bitmap>,
+	pub validity: Option<NullBuffer>,
 }
 
 impl Data {
@@ -43,15 +44,15 @@ impl Data {
 	}
 }
 
-impl From<mutable::Data> for Data {
-	fn from(d: mutable::Data) -> Self {
-		Self {
-			pre: d.pre.into(),
-			post: d.post.into(),
-			validity: d.validity.map(|v| v.into()),
-		}
-	}
-}
+//impl From<mutable::Data> for Data {
+//	fn from(d: mutable::Data) -> Self {
+//		Self {
+//			pre: d.pre.finish(),
+//			post: d.post.finish(),
+//			validity: d.validity.map(|v| v.into()),
+//		}
+//	}
+//}
 
 /// Frame data for a single port.
 #[derive(Debug)]
@@ -72,28 +73,28 @@ impl PortData {
 	}
 }
 
-impl From<mutable::PortData> for PortData {
-	fn from(p: mutable::PortData) -> Self {
-		Self {
-			port: p.port,
-			leader: p.leader.into(),
-			follower: p.follower.map(|f| f.into()),
-		}
-	}
-}
+//impl From<mutable::PortData> for PortData {
+//	fn from(p: mutable::PortData) -> Self {
+//		Self {
+//			port: p.port,
+//			leader: p.leader.finish(),
+//			follower: p.follower.map(|f| f.finish()),
+//		}
+//	}
+//}
 
 /// All frame data for a single game, in struct-of-arrays format.
 pub struct Frame {
 	/// Frame IDs start at `-123` and increment each frame. May repeat in case of rollbacks
-	pub id: PrimitiveArray<i32>,
+	pub id: PrimitiveArray<Int32Type>,
 	/// Port-specific data
 	pub ports: Vec<PortData>,
 	/// Start-of-frame data
 	pub start: Option<Start>,
 	/// End-of-frame data
 	pub end: Option<End>,
 	/// Logically, each frame has its own array of items. But we represent all item data in a flat array, with this field indicating the start of each sub-array
-	pub item_offset: Option<OffsetsBuffer<i32>>,
+	pub item_offset: Option<OffsetBuffer<i32>>,
 	/// Item data
 	pub item: Option<Item>,
 }
@@ -114,8 +115,8 @@ impl Frame {
 				self.end.as_ref().unwrap().transpose_one(i, version),
 			),
 			items: version.gte(3, 0).then(|| {
-				let (start, end) = self.item_offset.as_ref().unwrap().start_end(i);
-				(start..end)
+				let [start, end] = (*self.item_offset.as_ref().unwrap())[i .. i+1] else { panic!() };
+				(usize::try_from(start).unwrap() .. usize::try_from(end).unwrap())
 					.map(|i| self.item.as_ref().unwrap().transpose_one(i, version))
 					.collect()
 			}),
@@ -128,14 +129,14 @@ impl Frame {
 	pub fn rollbacks(&self, keep: Rollbacks) -> Vec<bool> {
 		use Rollbacks::*;
 		match keep {
-			ExceptFirst => self.rollbacks_(self.id.values_iter().enumerate()),
-			ExceptLast => self.rollbacks_(self.id.values_iter().enumerate().rev()),
+			ExceptFirst => self.rollbacks_(self.id.values().iter().cloned().enumerate()),
+			ExceptLast => self.rollbacks_(self.id.values().iter().cloned().enumerate().rev()),
 		}
 	}
 
-	fn rollbacks_<'a>(&self, ids: impl Iterator<Item = (usize, &'a i32)>) -> Vec<bool> {
+	fn rollbacks_<'a>(&self, ids: impl Iterator<Item = (usize, i32)>) -> Vec<bool> {
 		let mut result = vec![false; self.len()];
-		let unique_id_count = self.id.values_iter().max().map_or(0, |idx| {
+		let unique_id_count = arrow::compute::kernels::aggregate::max(&self.id).map_or(0, |idx| {
 			1 + usize::try_from(idx - frame::FIRST_INDEX).unwrap()
 		});
 		let mut seen = vec![false; unique_id_count];
@@ -152,20 +153,20 @@ impl Frame {
 	}
 }
 
-impl From<mutable::Frame> for Frame {
-	fn from(f: mutable::Frame) -> Self {
-		Self {
-			id: f.id.into(),
-			ports: f.ports.into_iter().map(|p| p.into()).collect(),
-			start: f.start.map(|x| x.into()),
-			end: f.end.map(|x| x.into()),
-			item_offset: f.item_offset.map(|x|
-				OffsetsBuffer::try_from(Buffer::from(x.into_inner())).unwrap()
-			),
-			item: f.item.map(|x| x.into()),
-		}
-	}
-}
+//impl From<mutable::Frame> for Frame {
+//	fn from(f: mutable::Frame) -> Self {
+//		Self {
+//			id: f.id.into(),
+//			ports: f.ports.into_iter().map(|p| p.into()).collect(),
+//			start: f.start.map(|x| x.into()),
+//			end: f.end.map(|x| x.into()),
+//			item_offset: f.item_offset.map(|x|
+//				OffsetBuffer::try_from(Buffer::from(x.into_inner())).unwrap()
+//			),
+//			item: f.item.map(|x| x.into()),
+//		}
+//	}
+//}
 
 impl fmt::Debug for Frame {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> {