diff --git a/bin/load-easylist.py b/bin/load-easylist.py new file mode 100644 index 00000000..a55bea99 --- /dev/null +++ b/bin/load-easylist.py @@ -0,0 +1,25 @@ +import os + +import psycopg2 +from psycopg2.extras import Json + + +with open("easylist.txt") as f: + rules = [line.strip() for line in f if line.strip()] + +conn = psycopg2.connect(os.getenv("DSN")) + +with conn: + with conn.cursor() as cur: + for rule in rules: + cur.execute( + """ + INSERT INTO objects(resource_name, parent_id, id, data) + VALUES (%s, %s, gen_random_uuid(), %s) + """, + ( + "record", + "/buckets/main-workspace/collections/easylist", + Json({"rule": rule}), + ), + ) diff --git a/kinto-remote-settings/src/kinto_remote_settings/changes/utils.py b/kinto-remote-settings/src/kinto_remote_settings/changes/utils.py index dc5c9e92..7998201f 100644 --- a/kinto-remote-settings/src/kinto_remote_settings/changes/utils.py +++ b/kinto-remote-settings/src/kinto_remote_settings/changes/utils.py @@ -3,19 +3,21 @@ from uuid import UUID from kinto.core import utils as core_utils +from kinto.core.storage import Filter, Sort +from kinto.core.utils import COMPARISON from pyramid.settings import aslist -def bound_limit(settings: dict, value: Optional[int]) -> int: +def bound_limit(settings: dict, value: Optional[int]) -> Optional[int]: """ ``_limit`` querystring value has to be within what is configured as pagination size and max storage fetching size (respectively defaults as `None` and 10000 in `kinto.core.DEFAULT_SETTINGS`) """ max_fetch_size = settings["storage_max_fetch_size"] - paginate_by = settings["paginate_by"] or max_fetch_size - max_limit = min(paginate_by, max_fetch_size) - return min(abs(value), max_limit) if value is not None else max_limit + # paginate_by = settings["paginate_by"] or max_fetch_size + # max_limit = min(paginate_by, max_fetch_size) + return min(abs(value), max_fetch_size) if value is not None else None def monitored_timestamps(request): @@ -83,3 +85,30 @@ def change_entry_id(request, http_host, bucket_id, collection_id): entry_id = str(UUID(identifier)) _CHANGES_ENTRIES_ID_CACHE[cache_key] = entry_id return _CHANGES_ENTRIES_ID_CACHE[cache_key] + + +def paginated(storage, **kwargs): + """Paginated results from `storage.list_all()`. + + :param kwargs: Passed through unchanged to `list_all()`. + """ + + sorting = [Sort("last_modified", -1)] + kwargs["sorting"] = sorting + object_pagination = None + while True: + objects = storage.list_all( + pagination_rules=object_pagination, **kwargs + ) + + if not objects: + break + + for obj in objects: + yield obj + + last = objects[-1] + + object_pagination = [ + [Filter(sorting[0].field, last[sorting[0].field], COMPARISON.LT), Filter('id', last['id'], COMPARISON.NOT)], + ] diff --git a/kinto-remote-settings/src/kinto_remote_settings/changes/views.py b/kinto-remote-settings/src/kinto_remote_settings/changes/views.py index 8b4f6393..1aab490e 100644 --- a/kinto-remote-settings/src/kinto_remote_settings/changes/views.py +++ b/kinto-remote-settings/src/kinto_remote_settings/changes/views.py @@ -25,7 +25,7 @@ CHANNEL_ID, MONITOR_BUCKET, ) -from .utils import bound_limit, change_entry_id, monitored_timestamps +from .utils import bound_limit, change_entry_id, monitored_timestamps, paginated DAY_IN_SECONDS = 24 * 60 * 60 @@ -438,16 +438,18 @@ def get_changeset(request): raise # Fetch list of changes. - changes = storage.list_all( - resource_name="record", - parent_id=collection_uri, - filters=filters, - limit=limit, - id_field="id", - modified_field="last_modified", - deleted_field="deleted", - sorting=[Sort("last_modified", -1)], - include_deleted=include_deleted, + changes = list( + paginated( + storage, + resource_name="record", + parent_id=collection_uri, + filters=filters, + limit=limit, + id_field="id", + modified_field="last_modified", + deleted_field="deleted", + include_deleted=include_deleted, + ) ) # Fetch current collection timestamp. records_timestamp = storage.resource_timestamp(