Add a VortexSession for scalar value deserialization

encodings/fastlanes/public-api.lock

@@ -494,7 +494,7 @@ pub fn vortex_fastlanes::FoRVTable::child(array: &vortex_fastlanes::FoRArray, id
 
 pub fn vortex_fastlanes::FoRVTable::child_name(_array: &vortex_fastlanes::FoRArray, idx: usize) -> alloc::string::String
 
-pub fn vortex_fastlanes::FoRVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
+pub fn vortex_fastlanes::FoRVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
 
 pub fn vortex_fastlanes::FoRVTable::dtype(array: &vortex_fastlanes::FoRArray) -> &vortex_array::dtype::DType
 

encodings/fastlanes/src/for/vtable/mod.rs

@@ -132,9 +132,9 @@ impl VTable for FoRVTable {
         dtype: &DType,
         _len: usize,
         _buffers: &[BufferHandle],
-        _session: &VortexSession,
+        session: &VortexSession,
     ) -> VortexResult<Self::Metadata> {
-        let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype)?;
+        let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?;
         Scalar::try_new(dtype.clone(), scalar_value)
     }
 

encodings/sequence/public-api.lock

@@ -124,7 +124,7 @@ pub fn vortex_sequence::SequenceVTable::child(_array: &vortex_sequence::Sequence
 
 pub fn vortex_sequence::SequenceVTable::child_name(_array: &vortex_sequence::SequenceArray, idx: usize) -> alloc::string::String
 
-pub fn vortex_sequence::SequenceVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
+pub fn vortex_sequence::SequenceVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
 
 pub fn vortex_sequence::SequenceVTable::dtype(array: &vortex_sequence::SequenceArray) -> &vortex_array::dtype::DType
 

encodings/sequence/src/array.rs

@@ -322,7 +322,7 @@ impl VTable for SequenceVTable {
         dtype: &DType,
         _len: usize,
         _buffers: &[BufferHandle],
-        _session: &VortexSession,
+        session: &VortexSession,
     ) -> VortexResult<Self::Metadata> {
         let prost =
             <ProstMetadata<ProstSequenceMetadata> as DeserializeMetadata>::deserialize(bytes)?;
@@ -336,6 +336,7 @@ impl VTable for SequenceVTable {
                 .as_ref()
                 .ok_or_else(|| vortex_err!("base required"))?,
             &DType::Primitive(ptype, NonNullable),
+            session,
         )?
         .as_primitive()
         .pvalue()
@@ -347,6 +348,7 @@ impl VTable for SequenceVTable {
                 .as_ref()
                 .ok_or_else(|| vortex_err!("multiplier required"))?,
             &DType::Primitive(ptype, NonNullable),
+            session,
         )?
         .as_primitive()
         .pvalue()

encodings/sparse/public-api.lock

@@ -122,7 +122,7 @@ pub fn vortex_sparse::SparseVTable::child(array: &vortex_sparse::SparseArray, id
 
 pub fn vortex_sparse::SparseVTable::child_name(_array: &vortex_sparse::SparseArray, idx: usize) -> alloc::string::String
 
-pub fn vortex_sparse::SparseVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
+pub fn vortex_sparse::SparseVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
 
 pub fn vortex_sparse::SparseVTable::dtype(array: &vortex_sparse::SparseArray) -> &vortex_array::dtype::DType
 

encodings/sparse/src/lib.rs

@@ -163,7 +163,7 @@ impl VTable for SparseVTable {
         dtype: &DType,
         _len: usize,
         buffers: &[BufferHandle],
-        _session: &VortexSession,
+        session: &VortexSession,
     ) -> VortexResult<Self::Metadata> {
         let prost_patches =
             <ProstMetadata<ProstPatchesMetadata> as DeserializeMetadata>::deserialize(bytes)?;
@@ -175,7 +175,7 @@ impl VTable for SparseVTable {
         }
         let scalar_bytes: &[u8] = &buffers[0].clone().try_to_host_sync()?;
 
-        let scalar_value = ScalarValue::from_proto_bytes(scalar_bytes, dtype)?;
+        let scalar_value = ScalarValue::from_proto_bytes(scalar_bytes, dtype, session)?;
         let fill_value = Scalar::try_new(dtype.clone(), scalar_value)?;
 
         Ok(SparseMetadata {

vortex-array/public-api.lock

@@ -834,7 +834,7 @@ pub fn vortex_array::arrays::ConstantVTable::child(_array: &vortex_array::arrays
 
 pub fn vortex_array::arrays::ConstantVTable::child_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
+pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
 
 pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType
 
@@ -11542,9 +11542,9 @@ pub fn vortex_array::scalar::ScalarValue::into_utf8(self) -> vortex_buffer::stri
 
 impl vortex_array::scalar::ScalarValue
 
-pub fn vortex_array::scalar::ScalarValue::from_proto(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<core::option::Option<Self>>
+pub fn vortex_array::scalar::ScalarValue::from_proto(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<Self>>
 
-pub fn vortex_array::scalar::ScalarValue::from_proto_bytes(bytes: &[u8], dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<core::option::Option<Self>>
+pub fn vortex_array::scalar::ScalarValue::from_proto_bytes(bytes: &[u8], dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<Self>>
 
 impl vortex_array::scalar::ScalarValue
 
@@ -12250,7 +12250,7 @@ impl vortex_array::scalar::Scalar
 
 pub fn vortex_array::scalar::Scalar::from_proto(value: &vortex_proto::scalar::Scalar, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self>
 
-pub fn vortex_array::scalar::Scalar::from_proto_value(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<Self>
+pub fn vortex_array::scalar::Scalar::from_proto_value(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self>
 
 impl vortex_array::scalar::Scalar
 
@@ -15980,7 +15980,7 @@ pub fn vortex_array::stats::StatsSet::merge_unordered(self, other: &Self, dtype:
 
 impl vortex_array::stats::StatsSet
 
-pub fn vortex_array::stats::StatsSet::from_flatbuffer<'a>(fb: &vortex_flatbuffers::array::ArrayStats<'a>, array_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<Self>
+pub fn vortex_array::stats::StatsSet::from_flatbuffer<'a>(fb: &vortex_flatbuffers::array::ArrayStats<'a>, array_dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self>
 
 impl core::clone::Clone for vortex_array::stats::StatsSet
 
@@ -16666,7 +16666,7 @@ pub fn vortex_array::arrays::ConstantVTable::child(_array: &vortex_array::arrays
 
 pub fn vortex_array::arrays::ConstantVTable::child_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
+pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
 
 pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType
 

vortex-array/src/arrays/constant/vtable/mod.rs

@@ -133,7 +133,7 @@ impl VTable for ConstantVTable {
         dtype: &DType,
         _len: usize,
         buffers: &[BufferHandle],
-        _session: &VortexSession,
+        session: &VortexSession,
     ) -> VortexResult<Self::Metadata> {
         vortex_ensure!(
             buffers.len() == 1,
@@ -144,7 +144,7 @@ impl VTable for ConstantVTable {
         let buffer = buffers[0].clone().try_to_host_sync()?;
         let bytes: &[u8] = buffer.as_ref();
 
-        let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype)?;
+        let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?;
         let scalar = Scalar::try_new(dtype.clone(), scalar_value)?;
 
         Ok(scalar)

vortex-array/src/scalar/proto.rs

@@ -167,8 +167,12 @@ impl Scalar {
     /// # Errors
     ///
     /// Returns an error if type validation fails.
-    pub fn from_proto_value(value: &pb::ScalarValue, dtype: &DType) -> VortexResult<Self> {
-        let scalar_value = ScalarValue::from_proto(value, dtype)?;
+    pub fn from_proto_value(
+        value: &pb::ScalarValue,
+        dtype: &DType,
+        session: &VortexSession,
+    ) -> VortexResult<Self> {
+        let scalar_value = ScalarValue::from_proto(value, dtype, session)?;
 
         Scalar::try_new(dtype.clone(), scalar_value)
     }
@@ -192,7 +196,7 @@ impl Scalar {
             .as_ref()
             .ok_or_else(|| vortex_err!(Serde: "Scalar missing value"))?;
 
-        let value: Option<ScalarValue> = ScalarValue::from_proto(pb_scalar_value, &dtype)?;
+        let value: Option<ScalarValue> = ScalarValue::from_proto(pb_scalar_value, &dtype, session)?;
 
         Scalar::try_new(dtype, value)
     }
@@ -207,9 +211,13 @@ impl ScalarValue {
     /// # Errors
     ///
     /// Returns an error if decoding or type validation fails.
-    pub fn from_proto_bytes(bytes: &[u8], dtype: &DType) -> VortexResult<Option<Self>> {
+    pub fn from_proto_bytes(
+        bytes: &[u8],
+        dtype: &DType,
+        session: &VortexSession,
+    ) -> VortexResult<Option<Self>> {
         let proto = pb::ScalarValue::decode(bytes)?;
-        Self::from_proto(&proto, dtype)
+        Self::from_proto(&proto, dtype, session)
     }
 
     /// Creates a [`ScalarValue`] from its [protobuf](pb::ScalarValue) representation.
@@ -220,7 +228,11 @@ impl ScalarValue {
     /// # Errors
     ///
     /// Returns an error if the protobuf value cannot be converted to the given [`DType`].
-    pub fn from_proto(value: &pb::ScalarValue, dtype: &DType) -> VortexResult<Option<Self>> {
+    pub fn from_proto(
+        value: &pb::ScalarValue,
+        dtype: &DType,
+        session: &VortexSession,
+    ) -> VortexResult<Option<Self>> {
         let kind = value
             .kind
             .as_ref()
@@ -242,7 +254,7 @@ impl ScalarValue {
             Kind::F64Value(v) => f64_from_proto(*v, dtype)?,
             Kind::StringValue(s) => string_from_proto(s, dtype)?,
             Kind::BytesValue(b) => bytes_from_proto(b, dtype)?,
-            Kind::ListValue(v) => list_from_proto(v, dtype)?,
+            Kind::ListValue(v) => list_from_proto(v, dtype, session)?,
         }))
     }
 }
@@ -415,14 +427,22 @@ fn bytes_from_proto(bytes: &[u8], dtype: &DType) -> VortexResult<ScalarValue> {
 }
 
 /// Deserialize a [`ScalarValue::List`] from a protobuf `ListValue`.
-fn list_from_proto(v: &ListValue, dtype: &DType) -> VortexResult<ScalarValue> {
+fn list_from_proto(
+    v: &ListValue,
+    dtype: &DType,
+    session: &VortexSession,
+) -> VortexResult<ScalarValue> {
     let element_dtype = dtype
         .as_list_element_opt()
         .ok_or_else(|| vortex_err!(Serde: "expected List dtype for ListValue, got {dtype}"))?;
 
     let mut values = Vec::with_capacity(v.values.len());
     for elem in v.values.iter() {
-        values.push(ScalarValue::from_proto(elem, element_dtype.as_ref())?);
+        values.push(ScalarValue::from_proto(
+            elem,
+            element_dtype.as_ref(),
+            session,
+        )?);
     }
 
     Ok(ScalarValue::List(values))
@@ -604,6 +624,7 @@ mod tests {
         let scalar_value = ScalarValue::from_proto(
             &pb_scalar_value,
             &DType::Primitive(PType::U64, Nullability::NonNullable),
+            &session(),
         )
         .unwrap();
         assert_eq!(
@@ -615,6 +636,7 @@ mod tests {
         let scalar_value_f16 = ScalarValue::from_proto(
             &pb_scalar_value,
             &DType::Primitive(PType::F16, Nullability::Nullable),
+            &session(),
         )
         .unwrap();
 
@@ -651,6 +673,7 @@ mod tests {
             let read_back = ScalarValue::from_proto(
                 &pb_value,
                 &DType::Primitive(PType::F16, Nullability::NonNullable),
+                &session(),
             )
             .unwrap();
 
@@ -729,7 +752,7 @@ mod tests {
 
         for (name, value, dtype) in exact_roundtrip_cases {
             let pb_value = ScalarValue::to_proto(value.as_ref());
-            let read_back = ScalarValue::from_proto(&pb_value, &dtype).unwrap();
+            let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
 
             let original_debug = format!("{value:?}");
             let roundtrip_debug = format!("{read_back:?}");
@@ -764,7 +787,7 @@ mod tests {
 
         for (name, value, dtype, expected) in unsigned_cases {
             let pb_value = ScalarValue::to_proto(Some(&value));
-            let read_back = ScalarValue::from_proto(&pb_value, &dtype).unwrap();
+            let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
 
             match read_back.as_ref() {
                 Some(ScalarValue::Primitive(pv)) => {
@@ -808,7 +831,7 @@ mod tests {
 
         for (name, value, dtype, expected) in signed_cases {
             let pb_value = ScalarValue::to_proto(Some(&value));
-            let read_back = ScalarValue::from_proto(&pb_value, &dtype).unwrap();
+            let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
 
             match read_back.as_ref() {
                 Some(ScalarValue::Primitive(pv)) => {
@@ -837,7 +860,8 @@ mod tests {
         assert_eq!(
             Scalar::from_proto_value(
                 &pb::ScalarValue::from(&v),
-                &DType::Primitive(PType::U64, Nullability::Nullable)
+                &DType::Primitive(PType::U64, Nullability::Nullable),
+                &session()
             )
             .unwrap(),
             Scalar::primitive(0u64, Nullability::Nullable)
@@ -852,7 +876,8 @@ mod tests {
         assert_eq!(
             Scalar::from_proto_value(
                 &pb::ScalarValue::from(&v),
-                &DType::Primitive(PType::I64, Nullability::Nullable)
+                &DType::Primitive(PType::I64, Nullability::Nullable),
+                &session()
             )
             .unwrap(),
             Scalar::primitive(0i64, Nullability::Nullable)

vortex-array/src/serde.rs

@@ -377,7 +377,7 @@ impl ArrayParts {
         if let Some(stats) = self.flatbuffer().stats() {
             decoded
                 .statistics()
-                .set_iter(StatsSet::from_flatbuffer(&stats, dtype)?.into_iter());
+                .set_iter(StatsSet::from_flatbuffer(&stats, dtype, session)?.into_iter());
         }
 
         Ok(decoded)

vortex-array/src/stats/flatbuffers.rs

@@ -7,6 +7,7 @@ use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
 use vortex_flatbuffers::WriteFlatBuffer;
 use vortex_flatbuffers::array as fba;
+use vortex_session::VortexSession;
 
 use crate::dtype::DType;
 use crate::dtype::Nullability;
@@ -113,6 +114,7 @@ impl StatsSet {
     pub fn from_flatbuffer<'a>(
         fb: &fba::ArrayStats<'a>,
         array_dtype: &DType,
+        session: &VortexSession,
     ) -> VortexResult<Self> {
         let mut stats_set = StatsSet::default();
 
@@ -142,7 +144,8 @@ impl StatsSet {
                     if let Some(max) = fb.max()
                         && let Some(stat_dtype) = stat_dtype
                     {
-                        let value = ScalarValue::from_proto_bytes(max.bytes(), &stat_dtype)?;
+                        let value =
+                            ScalarValue::from_proto_bytes(max.bytes(), &stat_dtype, session)?;
                         let Some(value) = value else {
                             continue;
                         };
@@ -161,7 +164,8 @@ impl StatsSet {
                     if let Some(min) = fb.min()
                         && let Some(stat_dtype) = stat_dtype
                     {
-                        let value = ScalarValue::from_proto_bytes(min.bytes(), &stat_dtype)?;
+                        let value =
+                            ScalarValue::from_proto_bytes(min.bytes(), &stat_dtype, session)?;
                         let Some(value) = value else {
                             continue;
                         };
@@ -193,7 +197,8 @@ impl StatsSet {
                     if let Some(sum) = fb.sum()
                         && let Some(stat_dtype) = stat_dtype
                     {
-                        let value = ScalarValue::from_proto_bytes(sum.bytes(), &stat_dtype)?;
+                        let value =
+                            ScalarValue::from_proto_bytes(sum.bytes(), &stat_dtype, session)?;
                         let Some(value) = value else {
                             continue;
                         };

vortex-file/public-api.lock

@@ -134,7 +134,7 @@ impl vortex_file::FileStatistics
 
 pub fn vortex_file::FileStatistics::dtypes(&self) -> &alloc::sync::Arc<[vortex_array::dtype::DType]>
 
-pub fn vortex_file::FileStatistics::from_flatbuffer<'a>(fb: &vortex_flatbuffers::footer::FileStatistics<'a>, file_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<Self>
+pub fn vortex_file::FileStatistics::from_flatbuffer<'a>(fb: &vortex_flatbuffers::footer::FileStatistics<'a>, file_dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self>
 
 pub fn vortex_file::FileStatistics::get(&self, field_idx: usize) -> (&vortex_array::stats::stats_set::StatsSet, &vortex_array::dtype::DType)