Skip to content

feat: add time-range indexes for trending/leaderboard queries #3740

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
QuantumExplorer wants to merge 1 commit into
base: v3.1-dev
Choose a base branch
from
Open

feat: add time-range indexes for trending/leaderboard queries #3740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/dapi-grpc/protos/platform/v0/platform.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,16 @@ message GetDocumentsRequest {
BETWEEN_EXCLUDE_RIGHT = 8;
IN = 9;
STARTS_WITH = 10;
// Time-range bucket selection (v1 / `GetDocumentsRequestV1` only — the
// v0 CBOR where surface is unaffected). The clause's `field` names a
// timestamp property covered by a `timeRange` index; the operand
// (`DocumentFieldValue.text`) is the selector `"newest"` or `"oldest"`.
// The server resolves it to a concrete equality on the bucket start
// using the current block time, and the verifier re-derives the same
// bucket from the quorum-signed response metadata time — so the proof
// is an ordinary index/count proof. See `timeRange` in the document
// meta-schema and `drive::query::resolve_time_range_bucket_clause`.
IN_TIME_RANGE = 11;
}

// Tagged scalar (or list) operand for a `WhereClause`. The
Expand Down
29 changes: 29 additions & 0 deletions packages/rs-dpp/schema/meta_schemas/document/v1/document-meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,35 @@
"rangeAverageable": {
"type": "boolean",
"description": "Syntactic sugar: `rangeAverageable: true` is shorthand for `rangeCountable: true` + `rangeSummable: true`. Requires `averageable` to be set."
},
"timeRange": {
"type": "object",
"properties": {
"on": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"description": "Name of the timestamp index property to bucket. Must be this index's first property and refer to a millisecond-timestamp field ($createdAt / $updatedAt / $transferredAt or a Date property)."
},
"range": {
"type": "integer",
"minimum": 1,
"description": "Length of each time range window, in milliseconds. Must be an exact multiple of `step`."
},
"step": {
"type": "integer",
"minimum": 1,
"description": "Interval between successive range starts, in milliseconds. When `range` > `step` the ranges overlap and a document is indexed under `range / step` bucket-start values (capped at 256)."
},
"origin": {
"type": "integer",
"minimum": 0,
"description": "Reference origin for range alignment, in milliseconds. Range starts are `origin + k * step`. Defaults to 0."
}
},
"required": ["on", "range", "step"],
"additionalProperties": false,
"description": "Buckets the first index property's timestamp into fixed-length, regularly-spaced (possibly overlapping) time ranges. The stored key is the range start (a u64 ms). Enables trending/leaderboard queries via TOP(timeRange(...)). Available from protocol version 12."
}
},
"required": [
Expand Down
96 changes: 96 additions & 0 deletions packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ use crate::consensus::basic::data_contract::{
#[cfg(feature = "validation")]
use crate::consensus::ConsensusError;
use crate::data_contract::document_type::index::Index;
#[cfg(feature = "validation")]
use crate::data_contract::document_type::index::TimeRangeTransform;
use crate::data_contract::document_type::index_level::IndexLevel;
use crate::data_contract::document_type::property::DocumentProperty;
#[cfg(feature = "validation")]
Expand Down Expand Up @@ -374,6 +376,67 @@ impl DocumentTypeV1 {
)));
}

// `timeRange` indexes bucket a timestamp into
// fixed-length ranges; they rely on the time-range
// query operand and the multi-entry index insertion
// path that only exist from protocol v12 onward.
if index.time_range.is_some() && platform_version.protocol_version < 12
{
return Err(ProtocolError::ConsensusError(Box::new(
UnsupportedFeatureError::new(
"time range index".to_string(),
platform_version.protocol_version,
)
.into(),
)));
}

// The time-range source must be a millisecond
// timestamp: a system timestamp ($createdAt /
// $updatedAt / $transferredAt) or a user `Date`
// property. Structural checks (first-property,
// range % step, overlap cap) already happened in
// `Index` parsing.
if let Some(transform) = &index.time_range {
let source = transform.source.as_str();
let is_system_timestamp = matches!(
source,
property_names::CREATED_AT
| property_names::UPDATED_AT
| property_names::TRANSFERRED_AT
);
if !is_system_timestamp {
match flattened_document_properties.get(source) {
Some(def)
if matches!(
def.property_type,
DocumentPropertyType::Date
) => {}
Some(def) => {
return Err(ProtocolError::ConsensusError(Box::new(
InvalidIndexPropertyTypeError::new(
name.to_owned(),
index.name.to_owned(),
source.to_owned(),
def.property_type.name(),
)
.into(),
)));
}
None => {
return Err(ProtocolError::ConsensusError(Box::new(
UndefinedIndexPropertyError::new(
name.to_owned(),
index.name.to_owned(),
source.to_owned(),
)
.into(),
)));
}
}
}
}

validation_operations.extend(std::iter::once(
ProtocolValidationOperation::DocumentTypeSchemaIndexValidation(
index.properties.len() as u64,
Expand Down Expand Up @@ -590,6 +653,39 @@ impl DocumentTypeV1 {
.transpose()?
.unwrap_or_default();

// All indices that share a first property must agree on its time-range
// transform: either every such index buckets it with the identical
// transform, or none do. Otherwise the merged index trie node for that
// first property would be ambiguous (bucketed for one index, plain for
// another), so we reject the contract up front.
#[cfg(feature = "validation")]
if full_validation {
let mut first_property_time_range: BTreeMap<&str, Option<&TimeRangeTransform>> =
BTreeMap::new();
for index in indices.values() {
let Some(first) = index.properties.first() else {
continue;
};
let transform = index.time_range.as_ref();
match first_property_time_range.get(first.name.as_str()) {
Some(existing) if *existing != transform => {
return Err(consensus_or_protocol_data_contract_error(
DataContractError::InvalidContractStructure(format!(
"indices that share the first property \"{}\" must agree on its \
timeRange transform: either all bucket it identically or none \
do",
first.name
)),
));
}
Some(_) => {}
None => {
first_property_time_range.insert(first.name.as_str(), transform);
}
}
}
}

let index_structure =
IndexLevel::try_from_indices(indices.values(), name, platform_version)?;

Expand Down
Loading
Loading