RMST-371: WIP - 100K records

bin/load-easylist.py

@@ -0,0 +1,25 @@
+import os
+
+import psycopg2
+from psycopg2.extras import Json
+
+
+with open("easylist.txt") as f:
+    rules = [line.strip() for line in f if line.strip()]
+
+conn = psycopg2.connect(os.getenv("DSN"))
+
+with conn:
+    with conn.cursor() as cur:
+        for rule in rules:
+            cur.execute(
+                """
+                INSERT INTO objects(resource_name, parent_id, id, data)
+                VALUES (%s, %s, gen_random_uuid(), %s)
+                """,
+                (
+                    "record",
+                    "/buckets/main-workspace/collections/easylist",
+                    Json({"rule": rule}),
+                ),
+            )

kinto-remote-settings/src/kinto_remote_settings/changes/utils.py

@@ -3,19 +3,21 @@
 from uuid import UUID
 
 from kinto.core import utils as core_utils
+from kinto.core.storage import Filter, Sort
+from kinto.core.utils import COMPARISON
 from pyramid.settings import aslist
 
 
-def bound_limit(settings: dict, value: Optional[int]) -> int:
+def bound_limit(settings: dict, value: Optional[int]) -> Optional[int]:
     """
     ``_limit`` querystring value has to be within what is configured as
     pagination size and max storage fetching size (respectively defaults
     as `None` and 10000 in `kinto.core.DEFAULT_SETTINGS`)
     """
     max_fetch_size = settings["storage_max_fetch_size"]
-    paginate_by = settings["paginate_by"] or max_fetch_size
-    max_limit = min(paginate_by, max_fetch_size)
-    return min(abs(value), max_limit) if value is not None else max_limit
+    # paginate_by = settings["paginate_by"] or max_fetch_size
+    # max_limit = min(paginate_by, max_fetch_size)
+    return min(abs(value), max_fetch_size) if value is not None else None
 
 
 def monitored_timestamps(request):
@@ -83,3 +85,30 @@ def change_entry_id(request, http_host, bucket_id, collection_id):
         entry_id = str(UUID(identifier))
         _CHANGES_ENTRIES_ID_CACHE[cache_key] = entry_id
     return _CHANGES_ENTRIES_ID_CACHE[cache_key]
+
+
+def paginated(storage, **kwargs):
+    """Paginated results from `storage.list_all()`.
+
+    :param kwargs: Passed through unchanged to `list_all()`.
+    """
+
+    sorting = [Sort("last_modified", -1)]
+    kwargs["sorting"] = sorting
+    object_pagination = None
+    while True:
+        objects = storage.list_all(
+           pagination_rules=object_pagination, **kwargs
+        )
+
+        if not objects:
+            break
+
+        for obj in objects:
+            yield obj
+
+        last = objects[-1]
+
+        object_pagination = [
+            [Filter(sorting[0].field, last[sorting[0].field], COMPARISON.LT), Filter('id', last['id'], COMPARISON.NOT)],
+        ]

kinto-remote-settings/src/kinto_remote_settings/changes/views.py

@@ -25,7 +25,7 @@
     CHANNEL_ID,
     MONITOR_BUCKET,
 )
-from .utils import bound_limit, change_entry_id, monitored_timestamps
+from .utils import bound_limit, change_entry_id, monitored_timestamps, paginated
 
 
 DAY_IN_SECONDS = 24 * 60 * 60
@@ -438,16 +438,18 @@ def get_changeset(request):
             raise
 
         # Fetch list of changes.
-        changes = storage.list_all(
-            resource_name="record",
-            parent_id=collection_uri,
-            filters=filters,
-            limit=limit,
-            id_field="id",
-            modified_field="last_modified",
-            deleted_field="deleted",
-            sorting=[Sort("last_modified", -1)],
-            include_deleted=include_deleted,
+        changes = list(
+            paginated(
+                storage,
+                resource_name="record",
+                parent_id=collection_uri,
+                filters=filters,
+                limit=limit,
+                id_field="id",
+                modified_field="last_modified",
+                deleted_field="deleted",
+                include_deleted=include_deleted,
+            )
         )
         # Fetch current collection timestamp.
         records_timestamp = storage.resource_timestamp(