diff --git a/packages/dapi-grpc/protos/platform/v0/platform.proto b/packages/dapi-grpc/protos/platform/v0/platform.proto index 506db668e52..7c73209aed4 100644 --- a/packages/dapi-grpc/protos/platform/v0/platform.proto +++ b/packages/dapi-grpc/protos/platform/v0/platform.proto @@ -598,6 +598,16 @@ message GetDocumentsRequest { BETWEEN_EXCLUDE_RIGHT = 8; IN = 9; STARTS_WITH = 10; + // Time-range bucket selection (v1 / `GetDocumentsRequestV1` only — the + // v0 CBOR where surface is unaffected). The clause's `field` names a + // timestamp property covered by a `timeRange` index; the operand + // (`DocumentFieldValue.text`) is the selector `"newest"` or `"oldest"`. + // The server resolves it to a concrete equality on the bucket start + // using the current block time, and the verifier re-derives the same + // bucket from the quorum-signed response metadata time — so the proof + // is an ordinary index/count proof. See `timeRange` in the document + // meta-schema and `drive::query::resolve_time_range_bucket_clause`. + IN_TIME_RANGE = 11; } // Tagged scalar (or list) operand for a `WhereClause`. The diff --git a/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json b/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json index f1f4729b47c..64c96b048fa 100644 --- a/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json +++ b/packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json @@ -465,6 +465,35 @@ "rangeAverageable": { "type": "boolean", "description": "Syntactic sugar: `rangeAverageable: true` is shorthand for `rangeCountable: true` + `rangeSummable: true`. Requires `averageable` to be set." + }, + "timeRange": { + "type": "object", + "properties": { + "on": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "Name of the timestamp index property to bucket. Must be this index's first property and refer to a millisecond-timestamp field ($createdAt / $updatedAt / $transferredAt or a Date property)." + }, + "range": { + "type": "integer", + "minimum": 1, + "description": "Length of each time range window, in milliseconds. Must be an exact multiple of `step`." + }, + "step": { + "type": "integer", + "minimum": 1, + "description": "Interval between successive range starts, in milliseconds. When `range` > `step` the ranges overlap and a document is indexed under `range / step` bucket-start values (capped at 256)." + }, + "origin": { + "type": "integer", + "minimum": 0, + "description": "Reference origin for range alignment, in milliseconds. Range starts are `origin + k * step`. Defaults to 0." + } + }, + "required": ["on", "range", "step"], + "additionalProperties": false, + "description": "Buckets the first index property's timestamp into fixed-length, regularly-spaced (possibly overlapping) time ranges. The stored key is the range start (a u64 ms). Enables trending/leaderboard queries via TOP(timeRange(...)). Available from protocol version 12." } }, "required": [ diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs index 61c591299f4..4500fc8524a 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs @@ -7,6 +7,8 @@ use crate::consensus::basic::data_contract::{ #[cfg(feature = "validation")] use crate::consensus::ConsensusError; use crate::data_contract::document_type::index::Index; +#[cfg(feature = "validation")] +use crate::data_contract::document_type::index::TimeRangeTransform; use crate::data_contract::document_type::index_level::IndexLevel; use crate::data_contract::document_type::property::DocumentProperty; #[cfg(feature = "validation")] @@ -374,6 +376,67 @@ impl DocumentTypeV1 { ))); } + // `timeRange` indexes bucket a timestamp into + // fixed-length ranges; they rely on the time-range + // query operand and the multi-entry index insertion + // path that only exist from protocol v12 onward. + if index.time_range.is_some() && platform_version.protocol_version < 12 + { + return Err(ProtocolError::ConsensusError(Box::new( + UnsupportedFeatureError::new( + "time range index".to_string(), + platform_version.protocol_version, + ) + .into(), + ))); + } + + // The time-range source must be a millisecond + // timestamp: a system timestamp ($createdAt / + // $updatedAt / $transferredAt) or a user `Date` + // property. Structural checks (first-property, + // range % step, overlap cap) already happened in + // `Index` parsing. + if let Some(transform) = &index.time_range { + let source = transform.source.as_str(); + let is_system_timestamp = matches!( + source, + property_names::CREATED_AT + | property_names::UPDATED_AT + | property_names::TRANSFERRED_AT + ); + if !is_system_timestamp { + match flattened_document_properties.get(source) { + Some(def) + if matches!( + def.property_type, + DocumentPropertyType::Date + ) => {} + Some(def) => { + return Err(ProtocolError::ConsensusError(Box::new( + InvalidIndexPropertyTypeError::new( + name.to_owned(), + index.name.to_owned(), + source.to_owned(), + def.property_type.name(), + ) + .into(), + ))); + } + None => { + return Err(ProtocolError::ConsensusError(Box::new( + UndefinedIndexPropertyError::new( + name.to_owned(), + index.name.to_owned(), + source.to_owned(), + ) + .into(), + ))); + } + } + } + } + validation_operations.extend(std::iter::once( ProtocolValidationOperation::DocumentTypeSchemaIndexValidation( index.properties.len() as u64, @@ -590,6 +653,39 @@ impl DocumentTypeV1 { .transpose()? .unwrap_or_default(); + // All indices that share a first property must agree on its time-range + // transform: either every such index buckets it with the identical + // transform, or none do. Otherwise the merged index trie node for that + // first property would be ambiguous (bucketed for one index, plain for + // another), so we reject the contract up front. + #[cfg(feature = "validation")] + if full_validation { + let mut first_property_time_range: BTreeMap<&str, Option<&TimeRangeTransform>> = + BTreeMap::new(); + for index in indices.values() { + let Some(first) = index.properties.first() else { + continue; + }; + let transform = index.time_range.as_ref(); + match first_property_time_range.get(first.name.as_str()) { + Some(existing) if *existing != transform => { + return Err(consensus_or_protocol_data_contract_error( + DataContractError::InvalidContractStructure(format!( + "indices that share the first property \"{}\" must agree on its \ + timeRange transform: either all bucket it identically or none \ + do", + first.name + )), + )); + } + Some(_) => {} + None => { + first_property_time_range.insert(first.name.as_str(), transform); + } + } + } + } + let index_structure = IndexLevel::try_from_indices(indices.values(), name, platform_version)?; diff --git a/packages/rs-dpp/src/data_contract/document_type/index/mod.rs b/packages/rs-dpp/src/data_contract/document_type/index/mod.rs index b7190c50b8e..f2f1207265c 100644 --- a/packages/rs-dpp/src/data_contract/document_type/index/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/index/mod.rs @@ -30,6 +30,9 @@ use std::sync::OnceLock; use std::{collections::BTreeMap, convert::TryFrom}; pub mod random_index; +pub mod time_range; + +pub use time_range::TimeRangeTransform; #[repr(u8)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd)] @@ -416,6 +419,14 @@ pub struct Index { /// `packages/rs-drive/src/drive/document/primary_key_tree_type.rs` /// picks the appropriate variant. pub range_summable: bool, + /// When set, the index's first property is a timestamp that is bucketed + /// into fixed-length, regularly-spaced (possibly overlapping) time + /// ranges. The stored key for that property is the range *start* (a + /// `u64` millisecond timestamp), and a single document is indexed under + /// every range whose window contains its timestamp. See + /// [`TimeRangeTransform`]. The named source must be this index's first + /// property. Available from protocol version 12. + pub time_range: Option, } impl Index { @@ -621,6 +632,7 @@ impl TryFrom<&[(Value, Value)]> for Index { // alongside `summable: "y"`) before the merge. let mut averageable: Option = None; let mut range_averageable = false; + let mut time_range: Option = None; for (key_value, value_value) in index_type_value_map { let key = key_value.to_str()?; @@ -846,6 +858,85 @@ impl TryFrom<&[(Value, Value)]> for Index { "rangeAverageable value must be a boolean".to_string(), ))?; } + "timeRange" => { + let time_range_map = + value_value + .as_map() + .ok_or(DataContractError::ValueWrongType( + "timeRange value should be a map".to_string(), + ))?; + + let mut source: Option = None; + let mut range_ms: Option = None; + let mut step_ms: Option = None; + let mut origin_ms: u64 = 0; + + for (tr_key_value, tr_value) in time_range_map { + let tr_key = tr_key_value + .to_str() + .map_err(|e| DataContractError::ValueDecodingError(e.to_string()))?; + match tr_key { + "on" => { + source = Some( + tr_value + .as_text() + .ok_or(DataContractError::ValueWrongType( + "timeRange.on should be a string".to_string(), + ))? + .to_owned(), + ); + } + "range" => { + range_ms = Some(tr_value.to_integer().map_err(|_| { + DataContractError::ValueWrongType( + "timeRange.range should be an integer".to_string(), + ) + })?); + } + "step" => { + step_ms = Some(tr_value.to_integer().map_err(|_| { + DataContractError::ValueWrongType( + "timeRange.step should be an integer".to_string(), + ) + })?); + } + "origin" => { + origin_ms = tr_value.to_integer().map_err(|_| { + DataContractError::ValueWrongType( + "timeRange.origin should be an integer".to_string(), + ) + })?; + } + other => { + return Err(DataContractError::InvalidContractStructure(format!( + "unexpected timeRange field: {}", + other + ))); + } + } + } + + let source = source.ok_or(DataContractError::InvalidContractStructure( + "timeRange requires an `on` field naming the source timestamp property" + .to_string(), + ))?; + let range_ms = range_ms.ok_or(DataContractError::InvalidContractStructure( + "timeRange requires a `range` field (range length in milliseconds)" + .to_string(), + ))?; + let step_ms = step_ms.ok_or(DataContractError::InvalidContractStructure( + "timeRange requires a `step` field (interval between range starts in \ + milliseconds)" + .to_string(), + ))?; + + time_range = Some(TimeRangeTransform { + source, + range_ms, + step_ms, + origin_ms, + }); + } "properties" => { let properties = value_value @@ -995,6 +1086,71 @@ impl TryFrom<&[(Value, Value)]> for Index { )); } + // A time-range transform buckets the index's *first* property. Validate + // the structural constraints that don't need document-type context here + // (the source must be a timestamp/Date field — which does need the + // schema — is checked in `try_from_schema`). + if let Some(transform) = &time_range { + // Overlapping ranges index a single document under several bucket + // keys, which is fundamentally incompatible with uniqueness and + // with contested resources. Reject both up front. + if unique { + return Err(DataContractError::InvalidContractStructure( + "a timeRange index cannot be unique: overlapping ranges index one document \ + under several bucket keys" + .to_string(), + )); + } + if contested_index.is_some() { + return Err(DataContractError::InvalidContractStructure( + "a timeRange index cannot be a contested resource".to_string(), + )); + } + if transform.step_ms == 0 { + return Err(DataContractError::InvalidContractStructure( + "timeRange.step must be greater than zero".to_string(), + )); + } + if transform.range_ms == 0 { + return Err(DataContractError::InvalidContractStructure( + "timeRange.range must be greater than zero".to_string(), + )); + } + if transform.range_ms % transform.step_ms != 0 { + return Err(DataContractError::InvalidContractStructure( + "timeRange.range must be an exact multiple of timeRange.step so the number \ + of overlapping ranges per document is deterministic" + .to_string(), + )); + } + let overlap = transform.range_ms / transform.step_ms; + if overlap > time_range::MAX_TIME_RANGE_OVERLAP_FACTOR { + return Err(DataContractError::InvalidContractStructure(format!( + "timeRange overlap factor (range / step = {}) exceeds the maximum of {}; \ + a smaller window or larger step is required to bound per-document index \ + entries", + overlap, + time_range::MAX_TIME_RANGE_OVERLAP_FACTOR + ))); + } + match index_properties.first() { + Some(first) if first.name == transform.source => {} + Some(first) => { + return Err(DataContractError::InvalidContractStructure(format!( + "timeRange.on (\"{}\") must name the first index property (\"{}\"); a \ + time range partitions the index by time, so it has to be the leading \ + property", + transform.source, first.name + ))); + } + None => { + return Err(DataContractError::InvalidContractStructure( + "an index with a timeRange must have at least one property".to_string(), + )); + } + } + } + // if the index didn't have a name let's make one let name = name.unwrap_or_else(|| Alphanumeric.sample_string(&mut rand::thread_rng(), 24)); @@ -1008,6 +1164,7 @@ impl TryFrom<&[(Value, Value)]> for Index { range_countable, summable, range_summable, + time_range, }) } } @@ -1065,9 +1222,106 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, } } + // ----------------------------------------------------------------------- + // timeRange parsing + structural validation tests + // ----------------------------------------------------------------------- + + fn index_value_map( + first_property: &str, + time_range: Option<(&str, u64, u64)>, + ) -> Vec<(Value, Value)> { + let property = Value::Map(vec![( + Value::Text(first_property.to_string()), + Value::Text("asc".to_string()), + )]); + let hashtag = Value::Map(vec![( + Value::Text("hashtag".to_string()), + Value::Text("asc".to_string()), + )]); + + let mut map = vec![ + ( + Value::Text("name".to_string()), + Value::Text("trending".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![property, hashtag]), + ), + ]; + + if let Some((on, range, step)) = time_range { + map.push(( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + (Value::Text("on".to_string()), Value::Text(on.to_string())), + (Value::Text("range".to_string()), Value::U64(range)), + (Value::Text("step".to_string()), Value::U64(step)), + ]), + )); + } + + map + } + + #[test] + fn time_range_index_parses() { + let map = index_value_map("$createdAt", Some(("$createdAt", 21_600_000, 7_200_000))); + let index = Index::try_from(map.as_slice()).expect("should parse"); + let transform = index.time_range.expect("time_range should be set"); + assert_eq!(transform.source, "$createdAt"); + assert_eq!(transform.range_ms, 21_600_000); + assert_eq!(transform.step_ms, 7_200_000); + assert_eq!(transform.origin_ms, 0); + assert_eq!(transform.overlap_factor(), 3); + } + + #[test] + fn time_range_rejects_non_multiple_range() { + let map = index_value_map("$createdAt", Some(("$createdAt", 21_600_000, 7_000_000))); + let err = Index::try_from(map.as_slice()).unwrap_err(); + assert!(matches!( + err, + DataContractError::InvalidContractStructure(_) + )); + } + + #[test] + fn time_range_rejects_overlap_over_cap() { + // overlap factor = 300 > 256 + let map = index_value_map("$createdAt", Some(("$createdAt", 300, 1))); + let err = Index::try_from(map.as_slice()).unwrap_err(); + assert!(matches!( + err, + DataContractError::InvalidContractStructure(_) + )); + } + + #[test] + fn time_range_rejects_source_not_first_property() { + // `on` names a property that is not the first index property + let map = index_value_map("$createdAt", Some(("hashtag", 60, 20))); + let err = Index::try_from(map.as_slice()).unwrap_err(); + assert!(matches!( + err, + DataContractError::InvalidContractStructure(_) + )); + } + + #[test] + fn time_range_rejects_zero_step() { + let map = index_value_map("$createdAt", Some(("$createdAt", 60, 0))); + let err = Index::try_from(map.as_slice()).unwrap_err(); + assert!(matches!( + err, + DataContractError::InvalidContractStructure(_) + )); + } + // ----------------------------------------------------------------------- // ContestedIndexResolution tests // ----------------------------------------------------------------------- diff --git a/packages/rs-dpp/src/data_contract/document_type/index/random_index.rs b/packages/rs-dpp/src/data_contract/document_type/index/random_index.rs index b9b1050a3f6..8ec2098b507 100644 --- a/packages/rs-dpp/src/data_contract/document_type/index/random_index.rs +++ b/packages/rs-dpp/src/data_contract/document_type/index/random_index.rs @@ -64,6 +64,7 @@ impl Index { range_countable: false, summable: None, range_summable: false, + time_range: None, }) } } diff --git a/packages/rs-dpp/src/data_contract/document_type/index/time_range.rs b/packages/rs-dpp/src/data_contract/document_type/index/time_range.rs new file mode 100644 index 00000000000..93d6869c90a --- /dev/null +++ b/packages/rs-dpp/src/data_contract/document_type/index/time_range.rs @@ -0,0 +1,200 @@ +#[cfg(feature = "serde-conversion")] +use serde::{Deserialize, Serialize}; + +/// Maximum allowed overlap factor (`range_ms / step_ms`) for a time-range +/// transform. This bounds the number of index entries a single document +/// produces, preventing a contract from inflating storage and processing +/// cost by declaring a huge window over a tiny step. +pub const MAX_TIME_RANGE_OVERLAP_FACTOR: u64 = 256; + +/// An index-level transform that buckets a timestamp index property into +/// fixed-length, regularly-spaced time ranges. +/// +/// A time range is identified by a single `u64`: the **start time of the +/// range** in milliseconds. Each range covers `[start, start + range_ms)`. +/// New ranges start every `step_ms`. When `range_ms > step_ms` the ranges +/// overlap, so a single timestamp falls into `range_ms / step_ms` ranges +/// (the "overlap factor") and a document is indexed under that many +/// bucket-start values. +/// +/// The canonical use case is "trending" leaderboards: index on +/// `(timeRange($createdAt), hashtag)` with `countable`, then query a single +/// bucket ordered by count. Overlapping ranges guarantee that, at any +/// instant, there is always an active range covering a near-full `range_ms` +/// window of history (see [`Self::oldest_active_start`]). +/// +/// This transform lives on the index definition only. At the GroveDB storage +/// layer a bucket start is an ordinary `u64` key segment (encoded exactly +/// like a `$createdAt` value), so existing index queries, count trees and +/// proofs apply unchanged — the only novelty is that one document produces +/// several index entries. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde-conversion", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde-conversion", serde(rename_all = "camelCase"))] +pub struct TimeRangeTransform { + /// The source timestamp index property this transform buckets, e.g. + /// `$createdAt`. Must be the first property of the index and must refer + /// to a millisecond-timestamp field (`$createdAt` / `$updatedAt` / + /// `$transferredAt` or a user `Date` property). + pub source: String, + /// Length of each range window, in milliseconds. Must be a positive + /// multiple of `step_ms`. + pub range_ms: u64, + /// Interval between successive range starts, in milliseconds. Must be + /// greater than zero. + pub step_ms: u64, + /// Reference origin for range alignment, in milliseconds. Range starts + /// are `origin_ms + k * step_ms` for `k = 0, 1, 2, …`. Defaults to `0`. + pub origin_ms: u64, +} + +impl TimeRangeTransform { + /// The number of overlapping ranges that contain any given instant, i.e. + /// the number of bucket-start values a single document is indexed under. + /// Equal to `range_ms / step_ms`. + /// + /// Returns `0` only for a malformed transform with `step_ms == 0`; callers + /// constructing from a validated contract never observe that. + pub fn overlap_factor(&self) -> u64 { + if self.step_ms == 0 { + return 0; + } + self.range_ms / self.step_ms + } + + /// The start of the most recent range that has begun at or before `t`, + /// i.e. the largest `origin_ms + k * step_ms` that is `<= t`. + /// + /// For `t < origin_ms` (no range has started yet) this saturates to + /// `origin_ms`. + pub fn most_recent_start(&self, t: u64) -> u64 { + if self.step_ms == 0 { + return self.origin_ms; + } + let elapsed = t.saturating_sub(self.origin_ms); + self.origin_ms + (elapsed / self.step_ms) * self.step_ms + } + + /// All bucket-start values whose range `[start, start + range_ms)` + /// contains the timestamp `t`. This is the set of index entries a + /// document with timestamp `t` must be written under. + /// + /// The result is sorted in descending order (newest range first) and has + /// exactly [`Self::overlap_factor`] elements, except near `origin_ms` + /// where fewer ranges have started. + pub fn containing_buckets(&self, t: u64) -> Vec { + let overlap = self.overlap_factor(); + if overlap == 0 { + return Vec::new(); + } + let newest = self.most_recent_start(t); + (0..overlap) + .filter_map(|j| { + let offset = j.checked_mul(self.step_ms)?; + newest.checked_sub(offset) + }) + .filter(|start| *start >= self.origin_ms) + .collect() + } + + /// The start of the newest range that is active at `now` (the freshest + /// started range). Querying this bucket returns documents from the latest + /// partial slice — between `0` and `step_ms` of history. + pub fn newest_active_start(&self, now: u64) -> u64 { + self.most_recent_start(now) + } + + /// The start of the oldest range still active at `now`. Its window + /// `[start, start + range_ms)` still contains `now`, so querying this + /// bucket returns a near-full trailing window of `~range_ms` of history + /// (between `range_ms - step_ms` and `range_ms`). This is the bucket to + /// query for "trending over the last range window". + pub fn oldest_active_start(&self, now: u64) -> u64 { + let overlap = self.overlap_factor(); + if overlap == 0 { + return self.origin_ms; + } + let newest = self.most_recent_start(now); + let back = (overlap - 1).saturating_mul(self.step_ms); + newest.saturating_sub(back).max(self.origin_ms) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn transform() -> TimeRangeTransform { + // range = 6h, step = 2h, origin = 0 → overlap factor 3. + TimeRangeTransform { + source: "$createdAt".to_string(), + range_ms: 6 * 3_600_000, + step_ms: 2 * 3_600_000, + origin_ms: 0, + } + } + + #[test] + fn overlap_factor_is_range_over_step() { + assert_eq!(transform().overlap_factor(), 3); + } + + #[test] + fn most_recent_start_floors_to_step_multiple() { + let t = transform(); + let h = 3_600_000; + // now = 7h → most recent start = 6h + assert_eq!(t.most_recent_start(7 * h), 6 * h); + // exactly on a boundary stays put + assert_eq!(t.most_recent_start(6 * h), 6 * h); + // before origin saturates to origin + assert_eq!(t.most_recent_start(0), 0); + } + + #[test] + fn containing_buckets_are_the_overlapping_ranges() { + let t = transform(); + let h = 3_600_000; + // doc at 7h belongs to ranges starting at 6h, 4h, 2h + assert_eq!(t.containing_buckets(7 * h), vec![6 * h, 4 * h, 2 * h]); + // every returned range actually contains the timestamp + for start in t.containing_buckets(7 * h) { + assert!(start <= 7 * h && 7 * h < start + t.range_ms); + } + } + + #[test] + fn containing_buckets_truncate_near_origin() { + let t = transform(); + let h = 3_600_000; + // doc at 3h: ranges starting at 2h and 0h (4h start would be in future) + assert_eq!(t.containing_buckets(3 * h), vec![2 * h, 0]); + } + + #[test] + fn newest_vs_oldest_active() { + let t = transform(); + let h = 3_600_000; + let now = 7 * h; + // newest active = freshest started range + assert_eq!(t.newest_active_start(now), 6 * h); + // oldest active = covers the full trailing window + assert_eq!(t.oldest_active_start(now), 2 * h); + // oldest active range still contains now + let oldest = t.oldest_active_start(now); + assert!(oldest <= now && now < oldest + t.range_ms); + } + + #[test] + fn origin_offset_shifts_alignment() { + let t = TimeRangeTransform { + source: "$createdAt".to_string(), + range_ms: 60, + step_ms: 20, + origin_ms: 5, + }; + // starts are 5, 25, 45, ... ; now=50 → most recent start 45 + assert_eq!(t.most_recent_start(50), 45); + assert_eq!(t.containing_buckets(50), vec![45, 25, 5]); + } +} diff --git a/packages/rs-dpp/src/data_contract/document_type/index_level/find_first_change.rs b/packages/rs-dpp/src/data_contract/document_type/index_level/find_first_change.rs index 6e7c77cb332..ebc76f9260a 100644 --- a/packages/rs-dpp/src/data_contract/document_type/index_level/find_first_change.rs +++ b/packages/rs-dpp/src/data_contract/document_type/index_level/find_first_change.rs @@ -108,4 +108,40 @@ impl IndexLevel { None } + + /// Time-range counterpart of [`Self::find_first_countability_change`]. + /// Recursively finds the first index path where the `time_range` + /// transform differs between two `IndexLevel` trees. The transform + /// dictates how many index entries each document produces and under + /// which bucket keys, so changing it after creation would leave already + /// stored documents indexed under stale buckets — it is immutable. + /// + /// Returns `None` if the transform is the same everywhere. + #[cfg(feature = "validation")] + pub(super) fn find_first_time_range_change(&self, new: &IndexLevel) -> Option { + if self.time_range() != new.time_range() { + let fmt = |t: Option<&super::TimeRangeTransform>| match t { + Some(t) => format!( + "Some(on: {:?}, range: {}, step: {}, origin: {})", + t.source, t.range_ms, t.step_ms, t.origin_ms + ), + None => "None".to_string(), + }; + return Some(format!( + "(timeRange: {} -> {})", + fmt(self.time_range()), + fmt(new.time_range()), + )); + } + + for (key, old_sub) in &self.sub_index_levels { + if let Some(new_sub) = new.sub_index_levels.get(key) { + if let Some(inner_path) = old_sub.find_first_time_range_change(new_sub) { + return Some(format!("{} -> {}", key, inner_path)); + } + } + } + + None + } } diff --git a/packages/rs-dpp/src/data_contract/document_type/index_level/mod.rs b/packages/rs-dpp/src/data_contract/document_type/index_level/mod.rs index 36d4a24c4ba..b13b0b66233 100644 --- a/packages/rs-dpp/src/data_contract/document_type/index_level/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/index_level/mod.rs @@ -7,6 +7,7 @@ use crate::consensus::basic::data_contract::DuplicateIndexError; use crate::consensus::basic::BasicError; use crate::consensus::ConsensusError; use crate::data_contract::document_type::index::IndexCountability; +use crate::data_contract::document_type::index::TimeRangeTransform; use crate::data_contract::document_type::index_level::IndexType::{ ContestedResourceIndex, NonUniqueIndex, UniqueIndex, }; @@ -99,6 +100,14 @@ pub struct IndexLevel { sub_index_levels: BTreeMap, /// did an index terminate at this level has_index_with_type: Option, + /// When set, the property reached at this level is a timestamp that is + /// bucketed into time ranges (see [`TimeRangeTransform`]). Only ever set + /// on a *first-property* node (a direct child of the root), because a + /// time-range transform must be its index's leading property. At + /// insert/delete/update time the document's timestamp for this property + /// is expanded into one key per overlapping range bucket instead of a + /// single key. Immutable after contract creation. + time_range: Option, /// unique level identifier level_identifier: u64, } @@ -112,6 +121,12 @@ impl IndexLevel { &self.sub_index_levels } + /// The time-range transform applied to the property reached at this + /// level, if any. Only set on first-property nodes. + pub fn time_range(&self) -> Option<&TimeRangeTransform> { + self.time_range.as_ref() + } + pub fn has_index_with_type(&self) -> Option<&IndexLevelTypeInfo> { // Was `Option` (Copy) before the v3 sum-tree // expansion added `summable: Option` to the struct, which @@ -213,6 +228,7 @@ impl IndexLevel { let mut index_level = IndexLevel { sub_index_levels: Default::default(), has_index_with_type: None, + time_range: None, level_identifier: 0, }; @@ -221,9 +237,9 @@ impl IndexLevel { for index_to_borrow in indices { let index = index_to_borrow.borrow(); let mut current_level = &mut index_level; - let mut properties_iter = index.properties.iter().peekable(); + let mut properties_iter = index.properties.iter().enumerate().peekable(); - while let Some(index_part) = properties_iter.next() { + while let Some((position, index_part)) = properties_iter.next() { current_level = current_level .sub_index_levels .entry(index_part.name.clone()) @@ -233,9 +249,21 @@ impl IndexLevel { level_identifier: counter, sub_index_levels: Default::default(), has_index_with_type: None, + time_range: None, } }); + // A time-range transform always targets the index's first + // property, so record it on that first-property node. Cross-index + // consistency (all indices sharing this first property must agree + // on the transform) is enforced in `try_from_schema`; here we + // simply record the transform for the indices that declare it. + if position == 0 { + if let Some(transform) = &index.time_range { + current_level.time_range = Some(transform.clone()); + } + } + // The last property if properties_iter.peek().is_none() { // This level already has been initialized. @@ -347,6 +375,20 @@ impl IndexLevel { ); } + // A time-range transform determines how many index entries each + // document produces and under which bucket keys. Changing it after + // creation would leave already-stored documents indexed under stale + // buckets, so it is immutable — reject any change. + if let Some(time_range_change_path) = self.find_first_time_range_change(new_indices) { + return SimpleConsensusValidationResult::new_with_error( + DataContractInvalidIndexDefinitionUpdateError::new( + document_type_name.to_string(), + time_range_change_path, + ) + .into(), + ); + } + SimpleConsensusValidationResult::new() } } @@ -375,6 +417,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -406,6 +449,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![ @@ -422,6 +466,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }, Index { name: "test2".to_string(), @@ -436,6 +481,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }, ]; @@ -476,6 +522,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }, Index { name: "test2".to_string(), @@ -490,6 +537,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }, ]; @@ -506,6 +554,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -544,6 +593,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -565,6 +615,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -609,6 +660,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -624,6 +676,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -662,6 +715,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -677,6 +731,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -715,6 +770,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -730,6 +786,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -768,6 +825,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -806,6 +864,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -821,6 +880,7 @@ mod tests { range_countable: true, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -859,6 +919,7 @@ mod tests { range_countable: true, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -874,6 +935,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -918,6 +980,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -939,6 +1002,7 @@ mod tests { range_countable: true, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = @@ -983,6 +1047,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let new_indices = vec![Index { @@ -1004,6 +1069,7 @@ mod tests { range_countable: false, summable: None, range_summable: false, + time_range: None, }]; let old_index_structure = diff --git a/packages/rs-drive-abci/src/query/document_query/v1/conversions.rs b/packages/rs-drive-abci/src/query/document_query/v1/conversions.rs index a61152b6171..70dde0e08e6 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/conversions.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/conversions.rs @@ -40,7 +40,7 @@ use dpp::platform_value::Value; use drive::query::{ HavingAggregate, HavingAggregateFunction, HavingClause, HavingOperator, HavingRanking, HavingRankingKind, HavingRightOperand, OrderClause, SelectFunction, SelectProjection, - WhereClause, WhereOperator, + TimeRangeSelector, WhereClause, WhereOperator, }; /// Map a wire-level [`ProtoWhereOperator`] discriminant onto @@ -51,7 +51,7 @@ use drive::query::{ pub(super) fn where_operator_from_proto(op: i32) -> Result { let proto_op = ProtoWhereOperator::try_from(op).map_err(|_| { QueryError::InvalidArgument(format!( - "unknown WhereOperator discriminant: {} (valid values: 0..=10, see \ + "unknown WhereOperator discriminant: {} (valid values: 0..=11, see \ `get_documents_request::WhereOperator`)", op )) @@ -68,9 +68,58 @@ pub(super) fn where_operator_from_proto(op: i32) -> Result WhereOperator::BetweenExcludeRight, ProtoWhereOperator::In => WhereOperator::In, ProtoWhereOperator::StartsWith => WhereOperator::StartsWith, + // IN_TIME_RANGE is not an engine operator: it's resolved to a concrete + // equality from authoritative block time before clause conversion (see + // `is_time_range_clause` / `time_range_clause_from_proto`), so it must + // never reach this mapping. + ProtoWhereOperator::InTimeRange => { + return Err(QueryError::InvalidArgument( + "IN_TIME_RANGE where clauses are resolved from block time before \ + operator conversion and must not be mixed into normal clause decoding" + .to_string(), + )) + } }) } +/// Whether a wire where clause is a time-range selection +/// (`operator == IN_TIME_RANGE`). The v1 handler partitions these out and +/// resolves them from authoritative block time via +/// [`time_range_clause_from_proto`]. +pub(super) fn is_time_range_clause(clause: &ProtoWhereClause) -> bool { + clause.operator == ProtoWhereOperator::InTimeRange as i32 +} + +/// Decode an `IN_TIME_RANGE` wire where clause into its `(field, selector)`. +/// The operand carries the selector as `DocumentFieldValue.text` +/// (`"newest"` or `"oldest"`). +pub(super) fn time_range_clause_from_proto( + clause: ProtoWhereClause, +) -> Result<(String, TimeRangeSelector), QueryError> { + let field = clause.field; + let selector_text = match clause.value.and_then(|v| v.variant) { + Some(document_field_value::Variant::Text(s)) => s, + _ => { + return Err(QueryError::InvalidArgument(format!( + "IN_TIME_RANGE clause on field '{}' must carry a text operand of \ + \"newest\" or \"oldest\"", + field + ))) + } + }; + let selector = match selector_text.as_str() { + "newest" => TimeRangeSelector::Newest, + "oldest" => TimeRangeSelector::Oldest, + other => { + return Err(QueryError::InvalidArgument(format!( + "IN_TIME_RANGE selector must be \"newest\" or \"oldest\", got \"{}\"", + other + ))) + } + }; + Ok((field, selector)) +} + /// Map a wire [`ProtoDocumentFieldValue`] onto a /// `dpp::platform_value::Value`. Schema-agnostic — variants map /// 1:1 by primitive type and recurse for `list` up to a depth of diff --git a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs index 1414ced3826..bd839b1651d 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs @@ -35,6 +35,7 @@ use crate::error::query::QueryError; use crate::error::Error; use crate::platform_types::platform::Platform; use crate::platform_types::platform_state::PlatformState; +use crate::platform_types::platform_state::PlatformStateV0Methods; use crate::query::response_metadata::CheckpointUsed; use crate::query::QueryValidationResult; use dapi_grpc::platform::v0::get_documents_request::get_documents_request_v0::Start as RequestV0Start; @@ -464,10 +465,75 @@ impl Platform { // wire-malformed HAVING (bad discriminant, missing // aggregate, …) starts surfacing as `InvalidArgument` // automatically. - let where_clauses = match conversions::where_clauses_from_proto(proto_where_clauses) { + // Partition out time-range (IN_TIME_RANGE) clauses. They are resolved + // to concrete equality clauses on the bucketed source field using the + // authoritative committed block time, so the rest of the v1 pipeline + // (routing, executors, proofs) treats them as ordinary equality + // lookups. The verifier re-derives the same bucket from the + // quorum-signed response metadata time, so the proof matches. + let (time_range_proto, normal_proto): (Vec<_>, Vec<_>) = proto_where_clauses + .into_iter() + .partition(conversions::is_time_range_clause); + + let mut where_clauses = match conversions::where_clauses_from_proto(normal_proto) { Ok(c) => c, Err(e) => return Ok(QueryValidationResult::new_with_error(e)), }; + + if !time_range_proto.is_empty() { + let block_time_ms = + match platform_state.last_committed_block_time_ms() { + Some(t) => t, + None => return Ok(QueryValidationResult::new_with_error(QueryError::Query( + QuerySyntaxError::Unsupported( + "a time range (IN_TIME_RANGE) query requires a committed block time" + .to_string(), + ), + ))), + }; + let contract_id: Identifier = check_validation_result_with_data!(data_contract_id + .clone() + .try_into() + .map_err(|_| QueryError::InvalidArgument( + "id must be a valid identifier (32 bytes long)".to_string() + ))); + let (_, contract_fetch_info) = self.drive.get_contract_with_fetch_info_and_fee( + contract_id.to_buffer(), + None, + true, + None, + platform_version, + )?; + let contract_fetch_info = check_validation_result_with_data!(contract_fetch_info + .ok_or(QueryError::Query(QuerySyntaxError::DataContractNotFound( + "contract not found when resolving a time range query", + )))); + let contract_ref = &contract_fetch_info.contract; + let doc_type = check_validation_result_with_data!(contract_ref + .document_type_for_name(document_type.as_str()) + .map_err(|_| QueryError::InvalidArgument(format!( + "document type {} not found for contract {}", + document_type, contract_id + )))); + for proto_wc in time_range_proto { + let (field, selector) = match conversions::time_range_clause_from_proto(proto_wc) { + Ok(parsed) => parsed, + Err(e) => return Ok(QueryValidationResult::new_with_error(e)), + }; + match drive::query::resolve_time_range_bucket_clause( + &field, + selector, + doc_type, + block_time_ms, + ) { + Ok(resolved) => where_clauses.push(resolved), + Err(drive::error::Error::Query(qe)) => { + return Ok(QueryValidationResult::new_with_error(QueryError::Query(qe))) + } + Err(e) => return Err(e.into()), + } + } + } let order_by_clauses = match conversions::order_clauses_from_proto(proto_order_by) { Ok(c) => c, Err(e) => return Ok(QueryValidationResult::new_with_error(e)), diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs index d1e74ecd39a..36b905fe40f 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs @@ -12,7 +12,9 @@ use crate::drive::document::unique_event_id; use crate::util::type_constants::DEFAULT_HASH_SIZE_U8; use crate::drive::Drive; -use crate::util::object_size_info::{DocumentAndContractInfo, DocumentInfoV0Methods, PathInfo}; +use crate::util::object_size_info::{ + DocumentAndContractInfo, DocumentInfoV0Methods, DriveKeyInfo, PathInfo, +}; use crate::error::fee::FeeError; use crate::error::Error; @@ -21,6 +23,7 @@ use crate::fees::op::LowLevelDriveOperation; use dpp::data_contract::accessors::v0::DataContractV0Getters; use dpp::data_contract::config::v0::DataContractConfigGettersV0; use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::data_contract::document_type::DocumentPropertyType; use crate::drive::document::paths::contract_document_type_path_vec; use dpp::version::PlatformVersion; @@ -187,36 +190,72 @@ impl Drive { let any_fields_null = document_top_field.is_empty(); let all_fields_null = document_top_field.is_empty(); - let mut index_path_info = if document_and_contract_info - .owned_document_info - .document_info - .is_document_size() - { - // This is a stateless operation - PathInfo::PathWithSizes(KeyInfoPath::from_known_owned_path(index_path)) - } else { - PathInfo::PathAsVec::<0>(index_path) + // Mirror the insert side's time-range fan-out: a time-range + // first-property node removes one index entry per overlapping + // range bucket the document's timestamp fell into. The buckets are + // recomputed deterministically from the document's stored + // timestamp, so they match exactly what insert wrote. + let index_keys: Vec = match sub_level.time_range() { + Some(transform) if !document_top_field.is_empty() => match &document_top_field { + DriveKeyInfo::KeySize(_) => { + let overlap = transform.overlap_factor().max(1) as usize; + vec![document_top_field.clone(); overlap] + } + other => { + let raw = match other { + DriveKeyInfo::Key(k) => k.as_slice(), + DriveKeyInfo::KeyRef(k) => *k, + DriveKeyInfo::KeySize(_) => unreachable!(), + }; + match DocumentPropertyType::decode_date_timestamp(raw) { + Some(timestamp) => transform + .containing_buckets(timestamp) + .into_iter() + .map(|start| { + DriveKeyInfo::Key(DocumentPropertyType::encode_date_timestamp( + start, + )) + }) + .collect(), + None => vec![document_top_field.clone()], + } + } + }, + _ => vec![document_top_field], }; - // we push the actual value of the index path - index_path_info.push(document_top_field)?; - // the index path is now something likeDataContracts/ContractID/Documents(1)/$ownerId/ - - self.remove_indices_for_index_level_for_contract_operations( - document_and_contract_info, - index_path_info, - sub_level, - any_fields_null, - all_fields_null, - value_tree_type, - &storage_flags, - previous_batch_operations, - estimated_costs_only_with_layer_info, - event_id, - transaction, - batch_operations, - platform_version, - )?; + for index_key in index_keys { + let mut index_path_info = if document_and_contract_info + .owned_document_info + .document_info + .is_document_size() + { + // This is a stateless operation + PathInfo::PathWithSizes(KeyInfoPath::from_known_owned_path(index_path.clone())) + } else { + PathInfo::PathAsVec::<0>(index_path.clone()) + }; + + // we push the actual value of the index path + index_path_info.push(index_key)?; + // the index path is now something likeDataContracts/ContractID/Documents(1)/$ownerId/ + + self.remove_indices_for_index_level_for_contract_operations( + document_and_contract_info, + index_path_info, + sub_level, + any_fields_null, + all_fields_null, + value_tree_type, + &storage_flags, + previous_batch_operations, + estimated_costs_only_with_layer_info, + event_id, + transaction, + batch_operations, + platform_version, + )?; + } } Ok(()) } diff --git a/packages/rs-drive/src/drive/document/insert/add_document_for_contract/mod.rs b/packages/rs-drive/src/drive/document/insert/add_document_for_contract/mod.rs index df06944d777..d12482102b2 100644 --- a/packages/rs-drive/src/drive/document/insert/add_document_for_contract/mod.rs +++ b/packages/rs-drive/src/drive/document/insert/add_document_for_contract/mod.rs @@ -62,3 +62,223 @@ impl Drive { } } } + +#[cfg(test)] +mod time_range_index_e2e_tests { + //! End-to-end coverage for time-range index fan-out: a single document is + //! indexed under every overlapping range bucket its `$createdAt` falls + //! into, those buckets are queryable by exact bucket start, and deletion + //! removes every entry. + use crate::config::DriveConfig; + use crate::drive::Drive; + use crate::query::DriveDocumentQuery; + use crate::util::object_size_info::DocumentInfo::DocumentRefInfo; + use crate::util::object_size_info::{DocumentAndContractInfo, OwnedDocumentInfo}; + use crate::util::storage_flags::StorageFlags; + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use dpp::block::block_info::BlockInfo; + use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::DataContractFactory; + use dpp::document::{Document, DocumentV0, DocumentV0Getters}; + use dpp::platform_value::{platform_value, Identifier, Value}; + use dpp::prelude::DataContract; + use dpp::tests::utils::generate_random_identifier_struct; + use dpp::version::PlatformVersion; + use std::collections::BTreeMap; + + const HOUR_MS: u64 = 3_600_000; + + /// A v12 `post` document type with a `(timeRange($createdAt, range=6h, + /// step=2h), hashtag)` countable index — i.e. trending hashtags over a + /// 6-hour window refreshed every 2 hours (overlap factor 3). + fn build_trending_contract() -> DataContract { + let factory = DataContractFactory::new(12).expect("factory"); + let index_map = vec![ + ( + Value::Text("name".to_string()), + Value::Text("trending".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({"hashtag": "asc"}), + ]), + ), + ( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + (Value::Text("range".to_string()), Value::U64(6 * HOUR_MS)), + (Value::Text("step".to_string()), Value::U64(2 * HOUR_MS)), + ]), + ), + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + ]; + + let document_schema = platform_value!({ + "type": "object", + "properties": { + "hashtag": {"type": "string", "maxLength": 63, "position": 0}, + }, + "required": ["hashtag"], + "indices": Value::Array(vec![Value::Map(index_map)]), + "additionalProperties": false, + }); + let schemas = platform_value!({ "post": document_schema }); + let owner_id = generate_random_identifier_struct(); + factory + .create_with_value_config(owner_id, 0, schemas, None, None) + .expect("create contract") + .data_contract_owned() + } + + /// Number of documents the `trending` index returns for an exact + /// `$createdAt == bucket` lookup. + fn count_in_bucket( + drive: &Drive, + contract: &DataContract, + bucket: u64, + platform_version: &PlatformVersion, + ) -> usize { + let document_type = contract.document_type_for_name("post").expect("post"); + let query_value = Value::Map(vec![( + Value::Text("where".to_string()), + Value::Array(vec![Value::Array(vec![ + Value::Text("$createdAt".to_string()), + Value::Text("==".to_string()), + Value::U64(bucket), + ])]), + )]); + let query = DriveDocumentQuery::from_value( + query_value, + contract, + document_type, + &DriveConfig::default(), + ) + .expect("build query"); + query + .execute_raw_results_no_proof(drive, None, None, platform_version) + .expect("query") + .0 + .len() + } + + #[test] + fn time_range_insert_fans_out_to_overlapping_buckets_and_delete_removes_them() { + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + let contract = build_trending_contract(); + + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("trending") + .expect("trending index") + .time_range + .clone() + .expect("time range transform"); + assert_eq!(transform.overlap_factor(), 3); + + // A document created at 7h+ falls into the ranges starting at 6h, 4h, 2h. + let created_at = 7 * HOUR_MS + 123_456; + let expected_buckets = transform.containing_buckets(created_at); + assert_eq!( + expected_buckets, + vec![6 * HOUR_MS, 4 * HOUR_MS, 2 * HOUR_MS] + ); + + let owner_bytes = rand::random::<[u8; 32]>(); + let document = Document::V0(DocumentV0 { + id: Identifier::from(rand::random::<[u8; 32]>()), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([("hashtag".to_string(), Value::Text("ibiza".to_string()))]), + created_at: Some(created_at), + ..Default::default() + }); + let document_id = document.id(); + + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + platform_version, + None, + ) + .expect("add document"); + + // The document is queryable under each of its 3 overlapping buckets. + for bucket in &expected_buckets { + assert_eq!( + count_in_bucket(&drive, &contract, *bucket, platform_version), + 1, + "document should be indexed under bucket {bucket}" + ); + } + // It is NOT stored under the raw timestamp (only under bucket starts)… + assert_eq!( + count_in_bucket(&drive, &contract, created_at, platform_version), + 0, + "document must be indexed under bucket starts, not the raw timestamp" + ); + // …nor under a range that does not contain it. + assert_eq!( + count_in_bucket(&drive, &contract, 0, platform_version), + 0, + "an unrelated bucket must be empty" + ); + + // Deleting the document removes every bucket entry. + drive + .delete_document_for_contract( + document_id, + &contract, + "post", + BlockInfo::default(), + true, + None, + platform_version, + None, + ) + .expect("delete document"); + + for bucket in &expected_buckets { + assert_eq!( + count_in_bucket(&drive, &contract, *bucket, platform_version), + 0, + "bucket {bucket} should be empty after deletion" + ); + } + } +} diff --git a/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v1/mod.rs index 57d9ef64665..40fffd0b614 100644 --- a/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v1/mod.rs @@ -4,7 +4,9 @@ use crate::util::type_constants::DEFAULT_HASH_SIZE_U8; use crate::util::grove_operations::BatchInsertTreeApplyType; use crate::drive::Drive; -use crate::util::object_size_info::{DocumentAndContractInfo, DocumentInfoV0Methods, PathInfo}; +use crate::util::object_size_info::{ + DocumentAndContractInfo, DocumentInfoV0Methods, DriveKeyInfo, PathInfo, +}; use crate::error::fee::FeeError; use crate::error::Error; @@ -12,6 +14,7 @@ use crate::fees::op::LowLevelDriveOperation; use dpp::data_contract::accessors::v0::DataContractV0Getters; use dpp::data_contract::config::v0::DataContractConfigGettersV0; use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::data_contract::document_type::DocumentPropertyType; use dpp::version::PlatformVersion; @@ -173,12 +176,11 @@ impl Drive { )? .unwrap_or_default(); - // The zero will not matter here, because the PathKeyInfo is variable - let path_key_info = document_top_field.clone().add_path::<0>(index_path.clone()); // here we are inserting the value tree (per distinct property value) // under the top-level property-name tree. The top-level property-name // tree itself is created at contract setup, so the apply_type's // `in_tree_type` reflects whichever variant the contract setup used. + // Same for every bucket key when this is a time-range node. let value_apply_type = if estimated_costs_only_with_layer_info.is_none() { BatchInsertTreeApplyType::StatefulBatchInsertTree } else { @@ -190,16 +192,6 @@ impl Drive { .unwrap_or_default(), } }; - self.batch_insert_empty_tree_if_not_exists( - path_key_info.clone(), - value_tree_type, - storage_flags, - value_apply_type, - transaction, - previous_batch_operations, - batch_operations, - drive_version, - )?; if let Some(estimated_costs_only_with_layer_info) = estimated_costs_only_with_layer_info { @@ -238,43 +230,95 @@ impl Drive { let any_fields_null = document_top_field.is_empty(); let all_fields_null = document_top_field.is_empty(); - let mut index_path_info = if document_and_contract_info - .owned_document_info - .document_info - .is_document_size() - { - // This is a stateless operation - PathInfo::PathWithSizes(KeyInfoPath::from_known_owned_path(index_path)) - } else { - PathInfo::PathAsVec::<0>(index_path) + // A time-range first-property node expands the document's single + // timestamp into one index entry per overlapping range bucket (the + // bucket *start*, encoded exactly like the timestamp). A normal + // property keeps its single key. Null timestamps are never bucketed + // (they keep their single null entry). + let index_keys: Vec = match sub_level.time_range() { + Some(transform) if !document_top_field.is_empty() => match &document_top_field { + // Estimated-cost path (size only): the real timestamp isn't + // available, so assume the worst case of `overlap_factor` + // overlapping buckets. + DriveKeyInfo::KeySize(_) => { + let overlap = transform.overlap_factor().max(1) as usize; + vec![document_top_field.clone(); overlap] + } + other => { + let raw = match other { + DriveKeyInfo::Key(k) => k.as_slice(), + DriveKeyInfo::KeyRef(k) => *k, + DriveKeyInfo::KeySize(_) => unreachable!(), + }; + match DocumentPropertyType::decode_date_timestamp(raw) { + Some(timestamp) => transform + .containing_buckets(timestamp) + .into_iter() + .map(|start| { + DriveKeyInfo::Key(DocumentPropertyType::encode_date_timestamp( + start, + )) + }) + .collect(), + None => vec![document_top_field.clone()], + } + } + }, + _ => vec![document_top_field], }; - // we push the actual value of the index path - index_path_info.push(document_top_field)?; - // the index path is now something likeDataContracts/ContractID/Documents(1)/$ownerId/ + for index_key in index_keys { + // The zero will not matter here, because the PathKeyInfo is variable + let path_key_info = index_key.clone().add_path::<0>(index_path.clone()); + self.batch_insert_empty_tree_if_not_exists( + path_key_info.clone(), + value_tree_type, + storage_flags, + value_apply_type, + transaction, + previous_batch_operations, + batch_operations, + drive_version, + )?; + + let mut index_path_info = if document_and_contract_info + .owned_document_info + .document_info + .is_document_size() + { + // This is a stateless operation + PathInfo::PathWithSizes(KeyInfoPath::from_known_owned_path(index_path.clone())) + } else { + PathInfo::PathAsVec::<0>(index_path.clone()) + }; - // Propagate the exact `value_tree_type` we just inserted - // forward as the recursive level's `parent_value_tree_type`. - // This carries the full per-axis kind (count / sum / both, - // each in its plain or provable variant) so the next - // level's wrapper-choice for continuation children picks - // the right wrapper variant - // (NonCounted / NotSummed / NotCountedOrSummed). - self.add_indices_for_index_level_for_contract_operations( - document_and_contract_info, - index_path_info, - sub_level, - any_fields_null, - all_fields_null, - value_tree_type, - previous_batch_operations, - &storage_flags, - estimated_costs_only_with_layer_info, - event_id, - transaction, - batch_operations, - platform_version, - )?; + // we push the actual value of the index path + index_path_info.push(index_key)?; + // the index path is now something likeDataContracts/ContractID/Documents(1)/$ownerId/ + + // Propagate the exact `value_tree_type` we just inserted + // forward as the recursive level's `parent_value_tree_type`. + // This carries the full per-axis kind (count / sum / both, + // each in its plain or provable variant) so the next + // level's wrapper-choice for continuation children picks + // the right wrapper variant + // (NonCounted / NotSummed / NotCountedOrSummed). + self.add_indices_for_index_level_for_contract_operations( + document_and_contract_info, + index_path_info, + sub_level, + any_fields_null, + all_fields_null, + value_tree_type, + previous_batch_operations, + &storage_flags, + estimated_costs_only_with_layer_info, + event_id, + transaction, + batch_operations, + platform_version, + )?; + } } Ok(()) } diff --git a/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v0/mod.rs index f50fadf27a7..f9ce38ac4a5 100644 --- a/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v0/mod.rs @@ -31,8 +31,11 @@ use crate::drive::document::paths::{ contract_documents_keeping_history_primary_key_path_for_document_id, contract_documents_primary_key_path, }; +use crate::util::object_size_info::DocumentInfo; use dpp::data_contract::document_type::methods::DocumentTypeBasicMethods; -use dpp::data_contract::document_type::{IndexCountability, IndexLevel}; +use dpp::data_contract::document_type::{ + DocumentPropertyType, DocumentTypeRef, Index, IndexCountability, IndexLevel, TimeRangeTransform, +}; use dpp::version::PlatformVersion; use grovedb::batch::key_info::KeyInfo; use grovedb::batch::key_info::KeyInfo::KnownKey; @@ -352,6 +355,31 @@ impl Drive { document_reference.clone() }; + // Time-range indexes store one entry per overlapping range bucket, + // so they need a set-diff update rather than the single old→new + // value transition below. `current_index_level` is still the + // top-level (source) node and `index_path` is the base + // (…//) at this point. + if let Some(transform) = &index.time_range { + self.update_time_range_index_for_contract_operations_v0( + index, + transform, + document, + &old_document_info, + owner_id, + document_type, + &index_path, + current_index_level, + &index_document_reference, + storage_flags, + previous_batch_operations, + &mut batch_operations, + transaction, + platform_version, + )?; + continue; + } + // with the example of the dashpay contract's first index // the index path is now something likeDataContracts/ContractID/Documents(1)/$ownerId let document_top_field = document @@ -706,4 +734,243 @@ impl Drive { } Ok(batch_operations) } + + /// Updates the index entries for a single **time-range** index. + /// + /// A time-range index stores one entry per overlapping range bucket the + /// document's timestamp falls into, so an update is a set diff rather than + /// the single old→new transition the normal-index path performs: + /// - buckets present only in the old timestamp's set are removed, + /// - buckets present only in the new timestamp's set are inserted, + /// - buckets present in both are refreshed, or (when a later index property + /// changed) deleted under the old sub-values and reinserted under the new + /// ones. + /// + /// Time-range indexes are validated to be non-unique and non-contested, so + /// only the non-unique terminator layout (`…//[0]/`) is + /// handled here. The estimated-cost / document-size update path never + /// reaches this method — it delegates to the insert path, which has its own + /// bucket fan-out. + #[allow(clippy::too_many_arguments)] + fn update_time_range_index_for_contract_operations_v0( + &self, + index: &Index, + transform: &TimeRangeTransform, + document: &Document, + old_document_info: &DocumentInfo, + owner_id: Option<[u8; 32]>, + document_type: DocumentTypeRef, + base_index_path: &[Vec], + top_index_level: &IndexLevel, + index_document_reference: &Element, + storage_flags: Option<&StorageFlags>, + previous_batch_operations: &mut Option<&mut Vec>, + batch_operations: &mut Vec, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let drive_version = &platform_version.drive; + + // New/old timestamps for the bucketed source property → bucket key sets. + let new_ts = document + .get_raw_for_document_type( + &transform.source, + document_type, + owner_id, + platform_version, + )? + .and_then(|bytes| DocumentPropertyType::decode_date_timestamp(&bytes)); + let old_ts = match old_document_info.get_raw_for_document_type( + &transform.source, + document_type, + None, + None, + platform_version, + )? { + Some(DriveKeyInfo::Key(k)) => DocumentPropertyType::decode_date_timestamp(&k), + Some(DriveKeyInfo::KeyRef(k)) => DocumentPropertyType::decode_date_timestamp(k), + _ => None, + }; + + let encode_buckets = |ts: Option| -> Vec> { + ts.map(|t| { + transform + .containing_buckets(t) + .into_iter() + .map(DocumentPropertyType::encode_date_timestamp) + .collect() + }) + .unwrap_or_default() + }; + let new_buckets = encode_buckets(new_ts); + let old_buckets = encode_buckets(old_ts); + + // Sub-property suffix (positions 1..) interleaved as [name, value, …] + // for both the new and old document, plus the IndexLevel node at each + // depth (for the matching tree variant on insert). + let mut levels: Vec<&IndexLevel> = Vec::new(); + let mut new_suffix: Vec> = Vec::new(); + let mut old_suffix: Vec> = Vec::new(); + let mut current_level = top_index_level; + for index_property in index.properties.iter().skip(1) { + current_level = + current_level + .sub_levels() + .get(&index_property.name) + .ok_or(Error::Drive(DriveError::CorruptedContractIndexes(format!( + "index structure missing sub_level '{}' under time-range index '{}'", + index_property.name, index.name + ))))?; + levels.push(current_level); + + let new_val = document + .get_raw_for_document_type( + &index_property.name, + document_type, + owner_id, + platform_version, + )? + .unwrap_or_default(); + let old_val = match old_document_info.get_raw_for_document_type( + &index_property.name, + document_type, + None, + None, + platform_version, + )? { + Some(DriveKeyInfo::Key(k)) => k, + Some(DriveKeyInfo::KeyRef(k)) => k.to_vec(), + _ => Vec::new(), + }; + new_suffix.push(index_property.name.as_bytes().to_vec()); + new_suffix.push(new_val); + old_suffix.push(index_property.name.as_bytes().to_vec()); + old_suffix.push(old_val); + } + + let suffix_changed = new_suffix != old_suffix; + let reference_tree_type = + reference_tree_type_for_index(index.countable, &index.summable, index.range_summable); + let top_value_tree_type = value_tree_type_for_index_level(top_index_level); + let doc_id = document.id(); + + let new_set: HashSet<&Vec> = new_buckets.iter().collect(); + let old_set: HashSet<&Vec> = old_buckets.iter().collect(); + + // Delete old entries: removed buckets, plus common buckets whose + // sub-values changed (their old-suffix path no longer matches). + for bucket in &old_buckets { + if new_set.contains(bucket) && !suffix_changed { + continue; // unchanged entry — refreshed below + } + let mut key_info_path: Vec = base_index_path + .iter() + .map(|s| KnownKey(s.clone())) + .collect(); + key_info_path.push(KnownKey(bucket.clone())); + for segment in &old_suffix { + key_info_path.push(KnownKey(segment.clone())); + } + key_info_path.push(KnownKey(vec![0])); + self.batch_delete_up_tree_while_empty( + KeyInfoPath::from_vec(key_info_path), + doc_id.as_slice(), + Some(CONTRACT_DOCUMENTS_PATH_HEIGHT), + BatchDeleteUpTreeApplyType::StatefulBatchDelete { + is_known_to_be_subtree_with_sum: Some(MaybeTree::NotTree), + }, + transaction, + previous_batch_operations, + batch_operations, + drive_version, + )?; + } + + // Insert/refresh new entries: added buckets are inserted; common + // buckets are refreshed when unchanged or reinserted when the suffix + // changed. + for bucket in &new_buckets { + if old_set.contains(bucket) && !suffix_changed { + // Unchanged entry — refresh the stored reference in place + // (its content can still differ via storage flags). + let mut path: Vec> = base_index_path.to_vec(); + path.push(bucket.clone()); + for segment in &new_suffix { + path.push(segment.clone()); + } + path.push(vec![0]); + self.batch_refresh_reference( + path, + doc_id.to_vec(), + index_document_reference.clone(), + storage_flags.is_none(), + batch_operations, + drive_version, + )?; + continue; + } + + // Materialize every tree along the entry path (mirrors the insert + // path's tree-variant dispatch), then store the reference. + let mut path: Vec> = base_index_path.to_vec(); + self.batch_insert_empty_tree_if_not_exists( + PathKeyInfo::PathKeyRef::<0>((path.clone(), bucket.as_slice())), + top_value_tree_type, + storage_flags, + BatchInsertTreeApplyType::StatefulBatchInsertTree, + transaction, + previous_batch_operations, + batch_operations, + drive_version, + )?; + path.push(bucket.clone()); + + for (i, level) in levels.iter().enumerate() { + let property_name = &new_suffix[i * 2]; + let value = &new_suffix[i * 2 + 1]; + self.batch_insert_empty_tree_if_not_exists( + PathKeyInfo::PathKeyRef::<0>((path.clone(), property_name.as_slice())), + property_name_tree_type_for_index_level(level), + storage_flags, + BatchInsertTreeApplyType::StatefulBatchInsertTree, + transaction, + previous_batch_operations, + batch_operations, + drive_version, + )?; + path.push(property_name.clone()); + self.batch_insert_empty_tree_if_not_exists( + PathKeyInfo::PathKeyRef::<0>((path.clone(), value.as_slice())), + value_tree_type_for_index_level(level), + storage_flags, + BatchInsertTreeApplyType::StatefulBatchInsertTree, + transaction, + previous_batch_operations, + batch_operations, + drive_version, + )?; + path.push(value.clone()); + } + + self.batch_insert_empty_tree_if_not_exists( + PathKeyInfo::PathKeyRef::<0>((path.clone(), &[0])), + reference_tree_type, + storage_flags, + BatchInsertTreeApplyType::StatefulBatchInsertTree, + transaction, + previous_batch_operations, + batch_operations, + drive_version, + )?; + path.push(vec![0]); + + self.batch_insert( + PathKeyRefElement::<0>((path, doc_id.as_slice(), index_document_reference.clone())), + batch_operations, + drive_version, + )?; + } + + Ok(()) + } } diff --git a/packages/rs-drive/src/query/drive_document_count_query/tests.rs b/packages/rs-drive/src/query/drive_document_count_query/tests.rs index ce21548ad2d..aed6c06482c 100644 --- a/packages/rs-drive/src/query/drive_document_count_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_count_query/tests.rs @@ -2054,6 +2054,7 @@ mod range_countable_picker_tests { // tree-shape resolver (see `primary_key_tree_type.rs`). summable: None, range_summable: false, + time_range: None, } } diff --git a/packages/rs-drive/src/query/drive_document_sum_query/tests.rs b/packages/rs-drive/src/query/drive_document_sum_query/tests.rs index 46a7deb18bc..8b03ed6520c 100644 --- a/packages/rs-drive/src/query/drive_document_sum_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_sum_query/tests.rs @@ -42,6 +42,7 @@ fn summable_index(name: &str, props: &[&str], summable: Option<&str>) -> Index { range_countable: false, summable: summable.map(String::from), range_summable: false, + time_range: None, } } @@ -58,6 +59,7 @@ fn range_summable_index(name: &str, props: &[&str], summable: &str) -> Index { range_countable: false, summable: Some(summable.to_string()), range_summable: true, + time_range: None, } } diff --git a/packages/rs-drive/src/query/mod.rs b/packages/rs-drive/src/query/mod.rs index 21895f8e16a..be50e920a5e 100644 --- a/packages/rs-drive/src/query/mod.rs +++ b/packages/rs-drive/src/query/mod.rs @@ -511,6 +511,65 @@ impl From for Vec { } } +/// Which active time range a `TOP(timeRange(...))` selection resolves to, +/// when the index's ranges overlap (`range > step`). Time-range queries are a +/// v1-only feature; the v0 query surface is unaffected. +#[cfg(any(feature = "server", feature = "verify"))] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum TimeRangeSelector { + /// The freshest started range (largest start ≤ now). Covers the latest + /// partial slice (0..step of history). + Newest, + /// The oldest range still active at now. Covers a near-full trailing + /// window of ~range of history. Best for "trending over the last window". + Oldest, +} + +/// Resolves a time-range selection on `field` into a concrete equality +/// [`WhereClause`] on the bucketed source field, using the index's +/// `timeRange` transform and an authoritative `block_time_ms`. +/// +/// The server supplies `block_time_ms` from current block time and the +/// verifier re-derives it from the quorum-signed response metadata `time_ms`, +/// so both produce the identical concrete equality query — the existing +/// index/count proofs apply unchanged and the engine never needs a dedicated +/// time-range operator. +#[cfg(any(feature = "server", feature = "verify"))] +pub fn resolve_time_range_bucket_clause( + field: &str, + selector: TimeRangeSelector, + document_type: DocumentTypeRef, + block_time_ms: u64, +) -> Result { + let transform = document_type + .indexes() + .values() + .find_map(|index| { + index + .time_range + .as_ref() + .filter(|transform| transform.source == field) + }) + .ok_or(Error::Query( + QuerySyntaxError::WhereClauseOnNonIndexedProperty(format!( + "no time-range index is defined on field \"{}\"", + field + )), + ))?; + + let bucket_start = match selector { + TimeRangeSelector::Newest => transform.newest_active_start(block_time_ms), + TimeRangeSelector::Oldest => transform.oldest_active_start(block_time_ms), + }; + + Ok(WhereClause { + field: field.to_string(), + operator: WhereOperator::Equal, + value: Value::U64(bucket_start), + }) +} + #[cfg(any(feature = "server", feature = "verify"))] /// Drive query struct #[derive(Debug, PartialEq, Clone)] diff --git a/packages/rs-sdk/src/platform/dashpay/contact_request_queries.rs b/packages/rs-sdk/src/platform/dashpay/contact_request_queries.rs index 2670ac96772..290e8f40719 100644 --- a/packages/rs-sdk/src/platform/dashpay/contact_request_queries.rs +++ b/packages/rs-sdk/src/platform/dashpay/contact_request_queries.rs @@ -49,6 +49,7 @@ impl Sdk { operator: WhereOperator::Equal, value: platform_value!(identity_id), }], + time_range_clauses: vec![], group_by: vec![], having: vec![], order_by_clauses: vec![], @@ -91,6 +92,7 @@ impl Sdk { operator: WhereOperator::Equal, value: platform_value!(identity_id), }], + time_range_clauses: vec![], group_by: vec![], having: vec![], order_by_clauses: vec![], diff --git a/packages/rs-sdk/src/platform/documents/document_query.rs b/packages/rs-sdk/src/platform/documents/document_query.rs index ecab8f26daf..82d41d4f88f 100644 --- a/packages/rs-sdk/src/platform/documents/document_query.rs +++ b/packages/rs-sdk/src/platform/documents/document_query.rs @@ -35,7 +35,7 @@ use dpp::{ use drive::query::{ DriveDocumentQuery, HavingAggregate, HavingAggregateFunction, HavingClause, HavingOperator, HavingRanking, HavingRankingKind, HavingRightOperand, InternalClauses, OrderClause, - SelectFunction, SelectProjection, WhereClause, WhereOperator, + SelectFunction, SelectProjection, TimeRangeSelector, WhereClause, WhereOperator, }; use drive_proof_verifier::{types::Documents, FromProof}; @@ -74,6 +74,14 @@ pub struct DocumentQuery { pub document_type_name: String, /// `where` clauses for the query pub where_clauses: Vec, + /// Time-range (`IN_TIME_RANGE`) selections — `(field, selector)` pairs on + /// a timestamp field covered by a `timeRange` index. These are emitted as + /// `IN_TIME_RANGE` clauses on the v1 wire and resolved server-side from + /// the current block time; the verifier re-derives the same bucket from + /// the quorum-signed response metadata time. v1-only (the v0 wire has no + /// `IN_TIME_RANGE` operator). See [`Self::with_time_range`]. + #[cfg_attr(feature = "mocks", serde(default))] + pub time_range_clauses: Vec<(String, TimeRangeSelector)>, /// SQL `GROUP BY` field names, in left-to-right order. Empty = /// no explicit grouping (aggregate count for `select=Count`). /// Only meaningful when `select=Count`; non-empty with @@ -123,6 +131,7 @@ impl DocumentQuery { data_contract: Arc::clone(&contract), document_type_name: document_type_name.to_string(), where_clauses: vec![], + time_range_clauses: vec![], group_by: Vec::new(), having: Vec::new(), order_by_clauses: vec![], @@ -175,6 +184,24 @@ impl DocumentQuery { self } + /// Restrict the query to a single time-range bucket of `field` + /// (a timestamp covered by a `timeRange` index), selecting either the + /// [`TimeRangeSelector::Newest`] or [`TimeRangeSelector::Oldest`] currently + /// active range. Emitted as an `IN_TIME_RANGE` clause on the v1 wire and + /// resolved server-side from the current block time; the proof verifier + /// re-derives the identical bucket from the quorum-signed response + /// metadata time. Requires Platform v3.1+ (v1 wire). + /// + /// Existing time-range selections are preserved. + pub fn with_time_range( + mut self, + field: impl Into, + selector: TimeRangeSelector, + ) -> Self { + self.time_range_clauses.push((field.into(), selector)); + self + } + /// Add order by clause to the query. /// /// Existing order by clauses will be preserved. @@ -330,7 +357,43 @@ impl FromProof for drive_proof_verifier::types::Documents { where Self: Sized + 'a, { - let request: Self::Request = request.into(); + let mut request: Self::Request = request.into(); + let response: Self::Response = response.into(); + + // A time-range (`IN_TIME_RANGE`) selection is resolved to a concrete + // bucket using the **quorum-signed** response metadata time — the same + // authoritative block time the server used to resolve it — so the + // reconstructed query matches the proof exactly. Resolve before the + // `DriveDocumentQuery` conversion so the engine sees ordinary equality + // clauses. + if !request.time_range_clauses.is_empty() { + let time_ms = response_metadata_time_ms(&response).ok_or( + drive_proof_verifier::Error::ResponseDecodeError { + error: "time range query proof response is missing block-time metadata" + .to_string(), + }, + )?; + let data_contract = Arc::clone(&request.data_contract); + let document_type = data_contract + .document_type_for_name(&request.document_type_name) + .map_err(|e| drive_proof_verifier::Error::RequestError { + error: format!("document type not found for time range query: {}", e), + })?; + let time_range_clauses = std::mem::take(&mut request.time_range_clauses); + for (field, selector) in time_range_clauses { + let resolved = drive::query::resolve_time_range_bucket_clause( + &field, + selector, + document_type, + time_ms, + ) + .map_err(|e| drive_proof_verifier::Error::RequestError { + error: format!("failed to resolve time range clause: {}", e), + })?; + request.where_clauses.push(resolved); + } + } + let drive_query: DriveDocumentQuery = (&request) .try_into() @@ -348,6 +411,17 @@ impl FromProof for drive_proof_verifier::types::Documents { } } +/// Extract the block time (`time_ms`) from a `GetDocumentsResponse`'s metadata, +/// across both the v0 and v1 response envelopes. Used to re-derive time-range +/// buckets from the quorum-signed metadata during proof verification. +fn response_metadata_time_ms(response: &platform_proto::GetDocumentsResponse) -> Option { + use platform_proto::get_documents_response::Version; + match response.version.as_ref()? { + Version::V0(v0) => v0.metadata.as_ref().map(|m| m.time_ms), + Version::V1(v1) => v1.metadata.as_ref().map(|m| m.time_ms), + } +} + /// Version-aware encoder. The dispatch is driven by the /// `drive_abci.query.document_query` feature-version on /// [`PlatformVersion`]: `0` → V0 wire (used by v3.0 testnet), `1` → @@ -369,6 +443,7 @@ impl TryFromPlatformVersioned for GetDocumentsRequest { data_contract, document_type_name, where_clauses, + time_range_clauses, group_by, having, order_by_clauses, @@ -390,21 +465,31 @@ impl TryFromPlatformVersioned for GetDocumentsRequest { ); match feature_version { - 0 => encode_v0( - data_contract.id().to_vec(), - document_type_name, - where_clauses, - order_by_clauses, - limit, - start, - &select, - &group_by, - &having, - ), + 0 => { + if !time_range_clauses.is_empty() { + return Err(Error::Config( + "time range (IN_TIME_RANGE) queries require Platform v3.1+ (the v1 \ + getDocuments wire); the v0 wire has no time-range operator" + .to_string(), + )); + } + encode_v0( + data_contract.id().to_vec(), + document_type_name, + where_clauses, + order_by_clauses, + limit, + start, + &select, + &group_by, + &having, + ) + } 1 => encode_v1( data_contract.id().to_vec(), document_type_name, where_clauses, + time_range_clauses, order_by_clauses, limit, start, @@ -426,6 +511,7 @@ fn encode_v1( data_contract_id: Vec, document_type: String, where_clauses: Vec, + time_range_clauses: Vec<(String, TimeRangeSelector)>, order_by_clauses: Vec, limit: u32, start: Option, @@ -433,10 +519,29 @@ fn encode_v1( group_by: Vec, having: Vec, ) -> Result { - let where_clauses = where_clauses + let mut where_clauses = where_clauses .into_iter() .map(where_clause_to_proto) .collect::, _>>()?; + // Append time-range selections as `IN_TIME_RANGE` clauses: field + + // `"newest"`/`"oldest"` text operand. The server resolves them to a + // concrete bucket from current block time; the verifier re-derives the + // same bucket from the signed response metadata time. + for (field, selector) in time_range_clauses { + let selector_text = match selector { + TimeRangeSelector::Newest => "newest", + TimeRangeSelector::Oldest => "oldest", + }; + where_clauses.push(ProtoWhereClause { + field, + operator: ProtoWhereOperator::InTimeRange as i32, + value: Some(ProtoDocumentFieldValue { + variant: Some(document_field_value::Variant::Text( + selector_text.to_string(), + )), + }), + }); + } let order_by = order_by_clauses .into_iter() .map(order_clause_to_proto) @@ -592,13 +697,14 @@ impl<'a> From<&'a DriveDocumentQuery<'a>> for DocumentQuery { }; Self { - // `DriveDocumentQuery` has no SELECT/GROUP BY/HAVING + // `DriveDocumentQuery` has no SELECT/GROUP BY/HAVING/time-range // concept — it's a documents-only query. Default to the // v1 documents shape. select: SelectProjection::documents(), data_contract: Arc::new(data_contract), document_type_name: document_type_name.to_string(), where_clauses, + time_range_clauses: Vec::new(), group_by: Vec::new(), having: Vec::new(), order_by_clauses, @@ -626,13 +732,14 @@ impl<'a> From> for DocumentQuery { }; Self { - // `DriveDocumentQuery` has no SELECT/GROUP BY/HAVING + // `DriveDocumentQuery` has no SELECT/GROUP BY/HAVING/time-range // concept — it's a documents-only query. Default to the // v1 documents shape. select: SelectProjection::documents(), data_contract: Arc::new(data_contract), document_type_name: document_type_name.to_string(), where_clauses, + time_range_clauses: Vec::new(), group_by: Vec::new(), having: Vec::new(), order_by_clauses, diff --git a/packages/rs-sdk/src/platform/dpns_usernames/mod.rs b/packages/rs-sdk/src/platform/dpns_usernames/mod.rs index cfb47694cef..4c02a147d80 100644 --- a/packages/rs-sdk/src/platform/dpns_usernames/mod.rs +++ b/packages/rs-sdk/src/platform/dpns_usernames/mod.rs @@ -396,6 +396,7 @@ impl Sdk { value: Value::Text(normalized_label), }, ], + time_range_clauses: vec![], group_by: vec![], having: vec![], order_by_clauses: vec![], @@ -465,6 +466,7 @@ impl Sdk { value: Value::Text(normalized_label), }, ], + time_range_clauses: vec![], group_by: vec![], having: vec![], order_by_clauses: vec![], diff --git a/packages/rs-sdk/src/platform/dpns_usernames/queries.rs b/packages/rs-sdk/src/platform/dpns_usernames/queries.rs index 9e17c8fa413..782fdcbb787 100644 --- a/packages/rs-sdk/src/platform/dpns_usernames/queries.rs +++ b/packages/rs-sdk/src/platform/dpns_usernames/queries.rs @@ -55,6 +55,7 @@ impl Sdk { operator: WhereOperator::Equal, value: Value::Identifier(identity_id.to_buffer()), }], + time_range_clauses: vec![], group_by: vec![], having: vec![], order_by_clauses: vec![], // Remove ordering by $createdAt as it might not be indexed @@ -141,6 +142,7 @@ impl Sdk { value: Value::Text(normalized_prefix), }, ], + time_range_clauses: vec![], group_by: vec![], having: vec![], order_by_clauses: vec![OrderClause { diff --git a/packages/wasm-sdk/src/dpns.rs b/packages/wasm-sdk/src/dpns.rs index 1911990c8ac..ab0e50cf98e 100644 --- a/packages/wasm-sdk/src/dpns.rs +++ b/packages/wasm-sdk/src/dpns.rs @@ -277,6 +277,7 @@ impl WasmSdk { operator: WhereOperator::Equal, value: Value::Identifier(identity_id.to_buffer()), }], + time_range_clauses: vec![], group_by: vec![], having: vec![], order_by_clauses: vec![], diff --git a/packages/wasm-sdk/src/queries/document.rs b/packages/wasm-sdk/src/queries/document.rs index 754cb933639..b882d93968c 100644 --- a/packages/wasm-sdk/src/queries/document.rs +++ b/packages/wasm-sdk/src/queries/document.rs @@ -10,7 +10,7 @@ use dash_sdk::drive::query::SelectProjection; use dash_sdk::platform::documents::document_query::DocumentQuery; use dash_sdk::platform::Fetch; use dash_sdk::platform::FetchMany; -use drive::query::{OrderClause, WhereClause, WhereOperator}; +use drive::query::{OrderClause, TimeRangeSelector, WhereClause, WhereOperator}; use drive_proof_verifier::{DocumentSplitAverages, DocumentSplitCounts, DocumentSplitSums}; use js_sys::Map; use serde::Deserialize; @@ -117,6 +117,20 @@ export interface DocumentsQuery { * @default [] */ groupBy?: string[]; + + /** + * Time-range bucket selections for "trending"-style queries. Each entry + * picks a single bucket of a timestamp field covered by a `timeRange` + * index. The server resolves the bucket from the current block time and + * the proof verifier re-derives it from the signed response metadata, so + * the result is provable. v1 / Platform v3.1+ only. + * + * - `selector: "oldest"` → the oldest still-active range (a near-full + * trailing window of ~`range`; best for "trending over the last window"). + * - `selector: "newest"` → the freshest started range (latest partial slice). + * @default [] + */ + timeRange?: { field: string; selector: "newest" | "oldest" }[]; } "#; @@ -152,6 +166,10 @@ struct DocumentsQueryInput { // `orderBy` field — the first clause's direction controls // split-mode entry ordering and `(In + prove)` walk order. No // separate `orderByAscending` knob. + /// Time-range bucket selections (`IN_TIME_RANGE`), each `{ field, + /// selector }`. v1-only; resolved server-side from block time. + #[serde(rename = "timeRange", default)] + time_range: Option>, } async fn build_documents_query( @@ -170,6 +188,7 @@ async fn build_documents_query( start_after, start_at, group_by: _, + time_range, } = input; let contract_id: Identifier = data_contract_id.into(); @@ -204,6 +223,13 @@ async fn build_documents_query( } } + if let Some(time_range_values) = time_range { + for clause_json in time_range_values.iter() { + let (field, selector) = parse_time_range_clause(clause_json)?; + query = query.with_time_range(field, selector); + } + } + if let Some(order_values) = order_by { for clause_json in order_values.iter() { let order_clause = parse_order_clause(clause_json)?; @@ -383,6 +409,33 @@ fn parse_where_clause(json_clause: &JsonValue) -> Result Result<(String, TimeRangeSelector), WasmSdkError> { + let object = json_clause.as_object().ok_or_else(|| { + WasmSdkError::invalid_argument("timeRange clause must be an object { field, selector }") + })?; + let field = object + .get("field") + .and_then(JsonValue::as_str) + .ok_or_else(|| { + WasmSdkError::invalid_argument("timeRange clause requires a string `field`") + })? + .to_string(); + let selector = match object.get("selector").and_then(JsonValue::as_str) { + Some("newest") => TimeRangeSelector::Newest, + Some("oldest") => TimeRangeSelector::Oldest, + _ => { + return Err(WasmSdkError::invalid_argument( + "timeRange clause `selector` must be \"newest\" or \"oldest\"", + )) + } + }; + Ok((field, selector)) +} + /// Parse JSON order by clause into OrderClause fn parse_order_clause(json_clause: &JsonValue) -> Result { let clause_array = json_clause